from model import Todo
import motor.motor_asyncio

client = motor.motor_asyncio.AsyncIOMotorClient('mongodb://root:example@localhost:27017/TodoList?authSource=admin')
database = client.TodoList
collection = database.todo

async def fetchOneTodo(title):
    document = await collection.find_one({"title":title})
    return document

async def fetchAllTodos():
    todos = []
    cursor = collection.find({})
    async for document in cursor:
        todos.append(Todo(**document))
    return todos

async def createTodo(todo):
    document = todo
    result = await collection.insert_one(document)
    return result

async def updateTodo(title,desc):
    await collection.update_one({"title":title}, {"$set":{
        "description":desc}})
    document = await collection.find_one({"title":title})
    return document

async def removeTodo(title):
    await collection.delete_one({"title":title})
    return True


import argparse
import torch
import os
from torch.utils.data import DataLoader
from dataset.dataset import Crowd
from model.model import Count
import numpy as np

def parse_arg():
    parser = argparse.ArgumentParser()
    parser.add_argument('--downsample-ratio', default=8, type=int,
                        help='the downsample ratio of the model')
    parser.add_argument('--data-dir', default='QNRF/val',
                        help='the directory of the data')

    parser.add_argument('--pretrained', default='pretrained/pcpvt_large.pth',
                        help='the path to the pretrained model')
    parser.add_argument('--model-path', default='history/lr_1e-4_gamma_2_15/best_model.pth',
                        help='the path to the model')
    parser.add_argument('--drop', type=float, default=0.0, metavar='PCT',
                        help='Dropout rate (default: 0.)')
    parser.add_argument('--drop-path', type=float, default=0.45, metavar='PCT',
                        help='Drop path rate (default: 0.1)')

    parser.add_argument('--device', default='0',
                        help="assign device")
    args = parser.parse_args()
    return args

if __name__ == '__main__':
    args = parse_arg()
    torch.backends.cudnn.benchmark = True
    os.environ['CUDA_VISIBLE_DEVICES'] = args.device.strip()
    dataset = Crowd(args.data_dir, 512, args.downsample_ratio, method='val')
    dataloader = DataLoader(dataset, 1, shuffle=False, pin_memory=False)
    model = Count(args)
    device = torch.device('cuda')
    model.to(device)
    model.load_state_dict(torch.load(args.model_path, device))
    model.eval()
    res = []
    step = 0
    for im, gt, size in dataloader:
        im = im.to(device)
        with torch.set_grad_enabled(False):
            result, _, _, _, _, _ = model(im)
            res1 = gt.item() - torch.sum(result).item()
            res.append(res1)
            print('{}/{}: GT:{}, predict:{:.2f}, diff:{:.2f}'.format(step, len(dataset), gt.item(), torch.sum(result).item(),
                                                             res1), size[0])
            step = step + 1
    print('MAE: {:.2f}, MSE:{:.2f}'.format(np.mean(np.abs(res)), np.sqrt(np.mean(np.square(res)))))


# -*- coding: utf-8 -*-

"""
简单的两步任务, 串行.
"""

from datetime import datetime
from airflow.decorators import dag, task

dag_id = "dag2"

@dag(
    dag_id=dag_id,
    start_date=datetime(2021, 1, 1),
    schedule="@once",
    catchup=False,
)
def dag2():
    """
    这个例子中我们有两个 Task, 先运行 Task 1, 再运行 Task 2. 如果 Task 1 不成功, 则不运行 Task 2.
    我们的两个 Task 的任务都是生成一个随机数, 然后写入到 S3 中. 不过 Task 1 有 50% 的概率会失败.
    你可以看到如果 Task 1 失败了, 则 Task 2 不会被执行.
    """
    task1_id = "task1"
    task2_id = "task2"

    @task(
        task_id=task1_id,
    )
    def task1():
        import json
        import random
        from datetime import datetime

        import boto3

        print("Start task1")

        # 有 50% 的概率失败
        if random.randint(1, 100) <= 50:
            raise Exception("Randomly failed")

        aws_account_id = boto3.client("sts").get_caller_identity()["Account"]
        aws_region = "us-east-1"

        value = random.randint(1, 100)
        print(f"Generated value is {value}")
        data = {
            "value": value,
            "datetime": datetime.now().isoformat(),
        }
        boto3.client("s3").put_object(
            Bucket=f"{aws_account_id}-{aws_region}-data",
            Key=f"projects/mwaa-poc/{dag_id}/{task1_id}.output.json",
            Body=json.dumps(data),
            ContentType="application/json",
        )

        print("End task1")
        return "Returned by task 1"

    @task(
        task_id=task2_id,
    )
    def task2():
        import json
        import random
        from datetime import datetime

        import boto3

        print("Start task2")

        aws_account_id = boto3.client("sts").get_caller_identity()["Account"]
        aws_region = "us-east-1"

        value = random.randint(1, 100)
        print(f"Generated value is {value}")
        data = {
            "value": value,
            "datetime": datetime.now().isoformat(),
        }
        boto3.client("s3").put_object(
            Bucket=f"{aws_account_id}-{aws_region}-data",
            Key=f"projects/mwaa-poc/{dag_id}/{task2_id}.output.json",
            Body=json.dumps(data),
            ContentType="application/json",
        )

        print("End task2")
        return "Returned by task 2"

    # 这里调用了两个 task 函数, 就相当于告诉 Airflow 我要运行他们两. 默认情况下 Airflow
    # 认为它们是并行, 没有依赖关系, 但是如果你用 >>, << 这样的符号连接他们, 就表示他们是
    # 有依赖关系的.
    # Ref: https://airflow.apache.org/docs/apache-airflow/stable/tutorial/fundamentals.html#setting-up-dependencies
    run_task1 = task1()
    run_task2 = task2()

    run_task1 >> run_task2

run_dag2 = dag2()


from rich import print
from rich.console import Console
from rich.table import Table
from decimal import *
import sys

# TODO: Load file by launching with arguments
#       Pitch calculation for non feed/rev
#       Ignore comments when finding G84

manualFile=False
filePath = "Sample Gcode/Solidworks/CGIP11719.nc"

if manualFile == True:
    with open(filePath, 'r') as file:
        lines = file.readlines()
else:
    try:
        droppedFile = sys.argv[1:]
    except IndexError:
        print("No file selected")

    for p in droppedFile:
        filePath = p
        with open(p, 'r') as file:
            lines = file.readlines()

def readLine(line):
    codeList = ['G','M','T','S','F','X','Y','Z','H','D','N','R','P']
    newWord = ""
    wordList = []
    comment = False
    for i, char in enumerate(line):
        if char == "(":
            comment=True
        elif char == ")" and comment == True:
            comment=False

        elif char in codeList or char == "\n" or char == " ":
            if newWord != "" and newWord != " ":
                wordList.append(newWord)
            newWord = ""
        if comment==False and char != ")":
            newWord += char
    return(wordList)

def readWords(words, values):
    for word in words:
        letter = word[0] # Separate key letter from value
        value = word[1:]

        if letter == 'F':
            if values["feed"] == False:
                values["feed"] = float(value)
        elif letter == 'S':
            if values["rpm"]== False:
                values["rpm"] = int(value)
        elif letter == 'T':
            if values["tool"] == False:
                values["tool"] = int(value)
            if values["feedPerRev"] == False:
                values["feedPerRev"] = "NO"
        elif letter == 'G' and int(value) == 98:
            values["feedPerRev"] = True

        # print((letter,value))
    return(values)

tappingCycleIndex = [] # Store all G84 values to read related Gcode
for lineIndex, line in enumerate(lines):
    if line.find("G84") != -1:
        tappingCycleIndex.append(lineIndex+1)

allValues = [] # Store information about each tapping cycle

for tappingCycle in tappingCycleIndex:
    valueDict = {"tool":False, "rpm":False, "feed":False, "feedPerRev":False}
    lineToRead = tappingCycle
    while False in valueDict.values():
        words = readLine(lines[lineToRead])
        readWords(words, valueDict)
        lineToRead -= 1
    allValues.append(valueDict)

# Sort data
allValues = sorted(allValues, key= lambda i: i['tool'])

# Calculate pitches
def calculatePitch(tappingData):
    if tappingData["feedPerRev"] == True:
        tappingData["metricPitch"] = round(tappingData["feed"] * 25.4, 3)
        tappingData["threadsPerInch"] = round(1 / tappingData["feed"], 2)
    else:
        tappingData["metricPitch"] = round(tappingData["rpm"] / tappingData["feed"] * 25.4, 3)
        tappingData["threadsPerInch"] = round(tappingData["rpm"] / tappingData["feed"], 2)

for value in allValues:
    calculatePitch(value)

# Print Results
print("File loaded:", filePath)
console = Console()
table = Table(show_header=True)
table.add_column("Tool #")
table.add_column("RPM")
table.add_column("Feed")
table.add_column("Metric Pitch")
table.add_column("Threads per Inch")
for tool in allValues:
    table.add_row(str(tool["tool"]),str(tool["rpm"]),str(tool["feed"]),str(tool["metricPitch"]),str(tool["threadsPerInch"]))
console.print(table)

input("Press enter to continue...")


import sys
from os import path
import numpy
import scipy
import cupy
import cupyx
import datetime
from dataclasses import dataclass

@dataclass
class ExecutionSetup:
    gpu: bool = False
    precision: str = 'float64'
    data_folder: str = f".{path.sep}data{path.sep}"
    precision_np = numpy.float64
    precision_cp = cupy.float64

class CPU_GPU_Abstractor:
    def __init__(self, exec_setup: ExecutionSetup):
        self.exec_setup = exec_setup
        self.__setup_numpy_and_scipy()
        self.__setup_precision()

        self.__pb_started = False

    def __setup_precision(self):
        if self.exec_setup.precision == 'float32':
            self.exec_setup.precision = self.xp.float32
            self.exec_setup.precision_np = numpy.float32
            self.exec_setup.precision_cp = cupy.float32
        elif self.exec_setup.precision == 'float64':
            self.exec_setup.precision = self.xp.float64
            self.exec_setup.precision_np = numpy.float64
            self.exec_setup.precision_cp = cupy.float64
        else:
            raise Exception("Invalid floating point precision specified")

    def __setup_numpy_and_scipy(self):
        if cupy.cuda.runtime.getDeviceCount() > 0 and self.exec_setup.gpu:
            self.xp = cupy
            self.scipy = cupyx.scipy
            self.gpu = True

            device = cupy.cuda.Device(0)
            print(f"GPU-Device ID: {device.id}")
            print(f"GPU-Compute Capability: {device.compute_capability}")
            print(f"GPU-Memory available: {device.mem_info[0]/1e6:.1f}/{device.mem_info[1]/1e6:.1f} MB")

        else:
            self.xp = numpy
            self.scipy = scipy
            self.gpu = False

    def _print_progress_bar(self, progress, max, progress_bar_length=40, title="Progress"):
        progress_ratio = progress/max
        if not self.__pb_started:
            self.__ts = datetime.datetime.now()
            self.__pb_started = True

        tc = datetime.datetime.now()
        tf = (1/progress_ratio -1)*(tc - self.__ts).total_seconds()

        title = f"\r{title}: {progress:5}/{max:5}: "
        success_rate = f" {progress_ratio*100:3.2f}%"
        remaining_time = f" ERT: {int(tf/60):02}:{int(tf%60):02}"
        number_of_progress_indicators = int(progress * progress_bar_length // (max))

        sys.stdout.write(title + "[" + number_of_progress_indicators*"#" + (progress_bar_length - number_of_progress_indicators)*"-" + "]" + success_rate + remaining_time)

    def _end_progress_bar(self):
        sys.stdout.write("\n")
        self.__pb_started = False

    def _get_array_from_disk_or_mem(self, A):
        if type(A) == str and path.exists(self.exec_setup.data_folder + A):
            return self.xp.load(self.exec_setup.data_folder + A)
        elif type(A) == cupy.ndarray or type(A) == numpy.ndarray:
            if self.gpu:
                return cupy.asarray(A)
            else:
                return cupy.asnumpy(A)
        else:
            raise Exception("Array has invalid type. Needs to be either a path to an array or a cupy.ndarray or a numpy.ndarray")

    def asnumpy(self, a):
        """Move array to host"""
        return cupy.asnumpy(a)

    def ascupy(self, a):
        """Move array to device"""
        return cupy.asarray(a)


from core.base_classes.entity import BaseEntityProps
from core.value_objects import DateVO, ID
from abc import ABC, abstractmethod
from typing import Any, Generic, TypeVar, get_args

from infrastructure.database.base_classes.mongodb import OrmEntityBase

Entity = TypeVar('Entity', bound=BaseEntityProps)
OrmEntity = TypeVar('OrmEntity')
Props = TypeVar('Props')

class OrmMapperBase(ABC, Generic[Entity, OrmEntity]):

    def __init__(self) -> None:
        super().__init__()

    @property
    @abstractmethod
    def entity_klass(self):
        # return get_args(self.__orig_bases__[0])[0]
        raise NotImplementedError()

    @property
    @abstractmethod
    def orm_entity_klass(self):
        # return get_args(self.__orig_bases__[0])[1]
        raise NotImplementedError()

    @abstractmethod
    def to_domain_props(self, orm_entity: OrmEntity) -> Any:
        return

    @abstractmethod
    def to_orm_props(self, entity: Entity) -> OrmEntity:
        return

    def to_domain_entity(self, orm_entity: OrmEntity) -> Entity:

        props = self.to_domain_props(orm_entity)

        return self.assign_props_to_entity(props, orm_entity)

    def to_orm_entity(self, entity: Entity) -> OrmEntity:

        props = self.to_orm_props(entity)

        return self.orm_entity_klass(**{
            **props,
            'id': entity.id.value,
            'created_at': entity.created_at.value,
            'updated_at': entity.updated_at.value
        })

    def assign_props_to_entity(
        self,
        entity_props: Any,
        orm_entity: OrmEntity
    ) -> Entity:

        return self.entity_klass.from_orm({
            **entity_props,
            "id": ID(str(orm_entity.id)),
            "created_at": DateVO(orm_entity.created_at),
            "updated_at": DateVO(orm_entity.updated_at)
        })


from rest_framework import serializers
from rest_framework.renderers import JSONRenderer
from rest_framework.response import Response
from rest_framework.views import APIView

from .models import Group, Facultie, Student, HistoryOfRating

class FacultiesSerializers(serializers.ModelSerializer):
    class Meta:
        model = Facultie
        fields = ('id', 'name', 'dean', 'phone_number')

class GroupsSerializers(serializers.ModelSerializer):
    faculty_name = serializers.SerializerMethodField('get_faculty_name')

    class Meta:
        model = Group
        fields = ('id', 'name', 'faculty_name', 'year')

    def get_faculty_name(self, Group):
        faculty_name = Group.faculty.name
        return faculty_name

class StudentsSerializers(serializers.ModelSerializer):
    class Meta:
        model = Student
        fields = ('id', 'name', 'email', 'rating', 'description', 'group')

class StudentProfileSerializers(serializers.ModelSerializer):
    group_name = serializers.SerializerMethodField('get_group_name')
    class Meta:
        model = Student
        fields = ('id', 'name', 'email', 'rating', 'description', 'group_name')

    def get_group_name(self, Student):
        group_name = Student.group.name
        return group_name

# class HistoryOfRatingSerializers(serializers.ModelSerializer):
#     class Meta:
#         model = HistoryOfRating
#         fields = ('date_of_change', 'reason', 'rating_value', 'id_student')

class HistoryOfRatingSerializers(serializers.ModelSerializer):
    class Meta:
        model = HistoryOfRating
        fields = ('date_of_change', 'reason', 'rating_value', 'id_student')

    def update(self, instance, validated_data):
        instance.rating_value = validated_data.get('rating_value', instance.rating_value)
        return instance

class ReasonsSerializer(APIView):
    REASONS = {'reasons': [{'reason': 'Пользование мобильным устройством во время пары', 'rating': '-20'},
                           {'reason': 'Опоздание более чем на 0 милисекунд', 'rating': '-50'},
                           {'reason': 'Недостаточная вежливость по отношению к преподавателю и студентам',
                            'rating': '-10'},
                           {'reason': 'Невнимательное поведение на лекции', 'rating': '-20'},
                           {'reason': 'Некрасивый почерк у доски', 'rating': '-10'},
                           {'reason': 'Недостаточная активность на семинаре или практике', 'rating': '-30'},
                           {'reason': 'Плохое настроение', 'rating': '-5'},
                           {'reason': 'Оскорбление администрации', 'rating': '-100'},
                           {'reason': 'Немытая голова', 'rating': '-10'},
                           {'reason': 'Задержка в сдаче дз', 'rating': '-50'},

                           {'reason': 'Хорошие взаимоотношения с администратором', 'rating': '+100'},
                           {'reason': 'Своевременное выполнение дз', 'rating': '+10'},
                           {'reason': 'Активное поведение на семинаре', 'rating': '+20'},
                           {'reason': 'Подарок преподавателю', 'rating': '+50'},
                           {'reason': 'Участие в мероприятии', 'rating': '+30'},
                           {'reason': 'Ответ у доски', 'rating': '+5'},
                           {'reason': 'Отсутствие пропусков за месяц', 'rating': '+10'},
                           {'reason': 'Участие в олимпиаде', 'rating': '+20'},
                           {'reason': 'Отсутствие долгов на протяжении года', 'rating': '+50'},
                           {'reason': 'Отсутствие опозданий за неделю', 'rating': '+5'}]}

    def get(self, request):
        return Response(self.REASONS)


import logging
import os

from django.conf import settings
from django.db.models import Q
from django.http import HttpResponse
from django.utils.safestring import mark_safe

from cvAppMain.helpers import get_cv_url
from cvAppMain.models import GeneratedPDF

logger = logging.getLogger('django')

def make_context(process, language):
    """Makes a context for CV generating"""
    context = {"process": process}
    for each in process.texts.filter(
            Q(language=language) |
            Q(language__isnull=True)):
        context[each.text_type.codename] = mark_safe(each.text)
    return context

def get_pdf(process, language):
    """Gets a cached PDF from DB or calls generating a new one"""
    saved_file = process.generatedpdf_set.filter(language=language).first()
    if saved_file:
        return saved_file.as_response()
    else:
        return render_to_pdf(process, language)

def render_to_pdf(process, language):
    """Uses WKHTMLTOPDF to generate a PDF from URL.
    This function requires PROTOCOL and HOST settings to be set."""
    temp_pdf = f'pdfs/{process.codename}_{language.lang}.pdf'
    wkhtmltopdf_command = f'wkhtmltopdf "{settings.PROTOCOL}{settings.HOST}{get_cv_url(process.codename, language, "html")}" {temp_pdf}'

    try:
        logger.info(f"Rendering PDF to:\n{temp_pdf}\nwith command:\n{wkhtmltopdf_command}\n")
        os.system(wkhtmltopdf_command)
    except Exception as e:
        logger.error("Error while generating PDF: " + str(e))

    try:
        saved_pdf = GeneratedPDF.objects.create(process=process, language=language, pdf=temp_pdf)
        return saved_pdf.as_response()
    except Exception as e:
        logger.error("Error while saving and returning PDF: " + str(e))
        return HttpResponse('Something went wrong while generating PDF! Sorry!')


from sklearn.decomposition import FastICA
import numpy
import matplotlib.pyplot as plt
import time
from scipy.io import wavfile
from Processor import Processor
from sklearn import preprocessing

class ICA:
    def __init__(self, mixed_sig, no_of_components):
        self.mixed_signal = mixed_sig
        self.num_components = no_of_components

    def train(self, opt, proc):
        print("Training ICA...")
        time_start = time.time()
        # whitening necessary

        if opt == 2:
            ica = FastICA(n_components=self.num_components, whiten=False)
        elif opt == 3:
            ica = FastICA(n_components=self.num_components, fun='exp')
        elif opt == 4:
            ica = FastICA(n_components=self.num_components, fun='cube')
        elif opt == 5:
            self.mixed_signal = preprocessing.scale(self.mixed_signal)
            ica = FastICA(n_components=self.num_components)
        else:
            ica = FastICA(n_components=self.num_components)

        # self.mixed_signal = preprocessing.scale(self.mixed_signal)
        reconstruct_signal = ica.fit_transform(self.mixed_signal)
        mixing_matrix = ica.mixing_

        time_stop = time.time() - time_start
        print("Training Complete under {} seconds".format(time_stop))

        if opt == 2:
            assert numpy.allclose(self.mixed_signal, numpy.dot(reconstruct_signal, mixing_matrix.T))
        else:
            assert numpy.allclose(self.mixed_signal, numpy.dot(reconstruct_signal, mixing_matrix.T) + ica.mean_)

        if opt == 2:
            remixed_mat = numpy.dot(reconstruct_signal, mixing_matrix.T)
        else:
            remixed_mat = numpy.dot(reconstruct_signal, mixing_matrix.T) + ica.mean_

        for q in range(len(remixed_mat.T)):
            y = remixed_mat.T[q] - self.mixed_signal.T[q]
            q = sum([i**2 for i in y])
            print("Residual value: {}".format(q))

        # for i in range(len(remixed_mat.T)):
        #     wavfile.write('aaa.wav', proc[0], numpy.asarray(remixed_mat.T[0], dtype=numpy.int16))
        return reconstruct_signal, mixing_matrix, remixed_mat

    def create_audio(self, reconstructed_mat, s_hat_names, remixed_mat, recon_names, rates):
        print("Creating recovered audio...")
        for count in range(len(s_hat)):
            wavfile.write(s_hat[count], rates[count], reconstructed_mat.T[count])
            wavfile.write(recon_names[count], rates[count], numpy.asarray(remixed_mat.T[count], dtype=numpy.int16))

    def plot_audio(self, mixed_sig, recovered_sig):
        print("Plotting graphs...")
        models = [mixed_sig, recovered_sig]
        names = ['Observations (mixed signal)',
                 'ICA recovered signals']
        colors = ['red', 'steelblue', 'orange', 'green', 'brown']

        plt.figure()

        for ii, (model, name) in enumerate(zip(models, names), 1):
            plt.subplot(2, 1, ii)
            plt.title(name)
            for sig, color in zip(model.T, colors):
                plt.plot(sig, color=color)

        plt.subplots_adjust(0.09, 0.04, 0.94, 0.94, 0.26, 0.46)
        plt.show()

if __name__ == '__main__':
    audios = ['mic1.wav', 'mic2.wav', 'mic3.wav', 'mic4.wav', 'mic5.wav']
    s_hat = ['shat1.wav', 'shat2.wav', 'shat3.wav', 'shat4.wav', 'shat5.wav']
    recon = ['recon1.wav', 'recon2.wav', 'recon3.wav', 'recon4.wav', 'recon5.wav']

    processor = Processor(audios)
    processor.generate_wav_data()

    # Options: 1: default, 2: whitening off, 3: exp function, 4: cube function, 5: centering
    option = 1
    ica_ = ICA(processor.mixed_signal, 5)
    recon_matrix, mixing_matrix, remixed = ica_.train(option, processor.rates)
    ica_.create_audio(recon_matrix, s_hat, remixed, recon, processor.rates)
    ica_.plot_audio(processor.mixed_signal, recon_matrix)


import os
import tarfile
from six.moves import urllib
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns
from sklearn.model_selection import train_test_split
from sklearn.pipeline import make_pipeline
from sklearn.preprocessing import StandardScaler
from sklearn.linear_model import LinearRegression
from sklearn.ensemble import RandomForestRegressor
from sklearn.metrics import mean_squared_error
# sns.set()

DOWNLOAD_ROOT = "https://raw.githubusercontent.com/ageron/handson-ml2/master/"
HOUSING_PATH = os.path.join("data", "housing")
HOUSING_URL = DOWNLOAD_ROOT + "datasets/housing/housing.tgz"

def fetch_housing_data(housing_url=HOUSING_URL, housing_path=HOUSING_PATH):
    if not os.path.isdir(housing_path):
        os.makedirs(housing_path)
    tgz_path = os.path.join(housing_path, "housing.tgz")
    urllib.request.urlretrieve(housing_url, tgz_path)
    housing_tgz = tarfile.open(tgz_path)
    housing_tgz.extractall(path=housing_path)
    housing_tgz.close()

def load_housing_data(housing_path=HOUSING_PATH):
    csv_path = os.path.join(housing_path, "housing.csv")
    return pd.read_csv(csv_path)

def get_data_info():
    print(housing.info())
    data_describe = housing.describe()
    return data_describe

fetch_housing_data()
housing = load_housing_data()
data_describe = get_data_info()
housing = pd.get_dummies(housing)

plt.figure(1)
sns.histplot(data=housing)

plt.figure(2)
sns.scatterplot(data=housing, x='longitude', y='latitude', hue='median_house_value', s=1)

corr_matrix = housing.corr()

housing["rooms_per_household"] = housing["total_rooms"]/housing["households"]
housing["bedrooms_per_room"] = housing["total_bedrooms"]/housing["total_rooms"]
housing["population_per_household"]=housing["population"]/housing["households"]

target = housing.pop('median_house_value')
housing['median_house_value'] = target

housing = housing.dropna()
corr_matrix_2 = housing.corr()

X = housing.loc[:, 'longitude':'population_per_household']
y =  housing['median_house_value']
X_train, X_test, y_train, y_test = train_test_split(X, y)

def evaluate_model(estimator):
    model = make_pipeline(StandardScaler(), estimator)
    model.fit(X_train, y_train)
    y_predicted = model.predict(X_train)
    print('\nAccuracy on train set:', model.score(X_train, y_train))
    mse = np.sqrt(mean_squared_error(y_train, y_predicted))
    print('Mean squared error:', mse)

    print('Accuracy on test set:', model.score(X_test, y_test))

estimators = [LinearRegression(), RandomForestRegressor()]
for i in estimators:
    evaluate_model(i)

plt.show()


import torch
from tqdm import tqdm, trange
from utils_squad_evaluate import EVAL_OPTS, main as evaluate_on_squad, plot_pr_curve
from utils_squad import (read_squad_examples, convert_examples_to_features,
                         RawResult, write_predictions,
                         RawResultExtended, write_predictions_extended)

def to_list(tensor):
    return tensor.detach().cpu().tolist()

def evaluate(model, validation_dataloader, features_val, examples_val, tokenizer, predict_file, len_val, batch_size, device):
  print("***** Running evaluation *****")
  print("  Num examples = %d" % len_val)
  print("  Batch size = %d" % batch_size)
  all_results = []
  for batch in tqdm(validation_dataloader, desc="Evaluating", miniters=100, mininterval=5.0):
    model.eval()
    batch = tuple(t.to(device) for t in batch)
    with torch.no_grad():
      inputs = {'input_ids':      batch[0],
                'attention_mask': batch[1],
                'token_type_ids': batch[2]
                }
      example_indices = batch[3]
      outputs = model(**inputs)

    for i, example_index in enumerate(example_indices):
      eval_feature = features_val[example_index.item()]
      unique_id = int(eval_feature.unique_id)

      result = RawResult(unique_id    = unique_id,
                         start_logits = to_list(outputs[0][i]),
                         end_logits   = to_list(outputs[1][i]))
      all_results.append(result)

  # Compute predictions
  output_prediction_file = "/predictions/predictions.json"
  output_nbest_file = "/predictions/nbest_predictions.json"
  output_null_log_odds_file = "/predictions/null_odds.json"
  output_dir = "/predictions/predict_results"

  write_predictions(examples_val, features_val, all_results, 10,
                  30, True, output_prediction_file,
                  output_nbest_file, output_null_log_odds_file, False,
                  True, 0.0)

  # Evaluate with the official SQuAD script
  evaluate_options = EVAL_OPTS(data_file=predict_file,
                               pred_file=output_prediction_file,
                               na_prob_file=output_null_log_odds_file,
                               out_image_dir=None)
  results = evaluate_on_squad(evaluate_options)
  return results


# -*- coding: utf-8 -*-
import requests
from bs4 import BeautifulSoup

def main():
    #--- 取得URLの決定
    url = 'https://github.com/librariapj'
    #--- requestsでdom取得
    response = requests.get(url)
    #--- 取得結果の表示
    print(response)
    # print(response.text)
    #--- BS4に取得内容を格納
    soup = BeautifulSoup(response.content, 'html.parser')
    #--- BS4の出力
    # 全文
    #print(soup.prettify())
    # aタグ
    print(soup.find_all("a"))

if __name__ == "__main__":
    main()


from django.core.paginator import Paginator
import plotly.graph_objects as go
from django.shortcuts import render
from django.contrib import messages
import matplotlib.dates as mdates
from .api_calls import (
    fetch_coins,
    fetch_market_charts,
    fetch_coin_details,
    fetch_order_book,
    fetch_coin_history,
    API_COINCAP_URL,
    API_BASE_URL,
    API_BASE_URL_CHARTS,
    API_BASE_URL_DETAILS,
    API_ORDER_BOOK_URL,
    params,
    chart_params,
    order_params
)
import matplotlib
matplotlib.use('Agg')
from dateutil.parser import parse as parse_date
import matplotlib.pyplot as plt
from io import BytesIO
import base64
from datetime import datetime
import re
import datetime

def crypto(request):
    # Fetch data
    coin_data = fetch_coins(API_BASE_URL, params)

    # Check if data was successfully fetch
    if coin_data is None:
        messages.error(request, 'The limit for fetch requests has been reached. Please try again later.')
        return render(request, 'crypto_pulse/crypto.html')

    if 'error' in coin_data and coin_data['error']['code'] == 429:
        messages.error(request, 'The limit for fetch requests has been reached. Please try again later.')
        return render(request, 'crypto_pulse/crypto.html')

    # Split data into 10 coins per page
    paginator = Paginator(coin_data, 10)
    page_number = request.GET.get('page')
    page_obj = paginator.get_page(page_number)

    # Store coin data in the session
    request.session['coin_data'] = coin_data

    # List to store coin data as dictionaries
    coins = []
    for coin in page_obj:
        coin_info = {
            'name': coin.get('name'),
            'symbol': coin.get('symbol'),
            'image': coin.get('image'),
            'market_cap': coin.get('market_cap'),
            'daily_price_change': coin.get('price_change_percentage_24h'),
            'daily_volume': coin.get('total_volume'),
            'current_price': coin.get('current_price'),
        }
        coins.append(coin_info)

    return render(request, 'crypto_pulse/crypto.html', {'page_obj': page_obj, 'coins': coins})

def coin_chart(request, coin_symbol):
    # Format symbol
    coin_symbol = coin_symbol.upper() + 'USDT'
    if coin_symbol == 'USDTUSDT':
        coin_symbol = 'BUSDUSDT'
    title_coin_symbol = coin_symbol.replace('USDT', '/USDT')

    # Fetch data
    candles_data = fetch_market_charts(API_BASE_URL_CHARTS, coin_symbol=coin_symbol, params=chart_params)
    orderbook_data = fetch_order_book(API_ORDER_BOOK_URL, coin_symbol=coin_symbol, params=order_params)

    # Check if data was successfully fetched
    if candles_data is None and orderbook_data is None:
        messages.error(request, 'Failed to fetch candlestick data. Please try again.')
        return render(request, 'crypto_pulse/coin_chart.html')

    # Parse the orderbook data from Binance API
    bids_data = orderbook_data['bids']
    asks_data = orderbook_data['asks']

    # Parse the data from the Binance API to create the candlestick chart
    timestamps = [candle[0] for candle in candles_data]
    open_prices = [float(candle[1]) for candle in candles_data]
    high_prices = [float(candle[2]) for candle in candles_data]
    low_prices = [float(candle[3]) for candle in candles_data]
    close_prices = [float(candle[4]) for candle in candles_data]

    # Calculate the date range for the last 3 months
    today = datetime.date.today()
    three_months_ago = today - datetime.timedelta(days=90)

    # Filter the data to show only the last 3 months
    filtered_data = [(timestamp, open_price, high_price, low_price, close_price)
                     for timestamp, open_price, high_price, low_price, close_price in zip(timestamps, open_prices, high_prices, low_prices, close_prices)
                     if datetime.datetime.fromtimestamp(timestamp / 1000).date() >= three_months_ago]

    # Create the candlestick chart
    fig = go.Figure(data=[go.Candlestick(x=[candle[0] for candle in filtered_data],
                                        open=[candle[1] for candle in filtered_data],
                                        high=[candle[2] for candle in filtered_data],
                                        low=[candle[3] for candle in filtered_data],
                                        close=[candle[4] for candle in filtered_data])])

    # Customize the chart layout
    fig.update_layout(
        xaxis=dict(
            type='date',  # Set the x-axis type to 'date'
            range=[three_months_ago, today + datetime.timedelta(days=1)],  # Set the x-axis range to include an extra day
            gridcolor='rgba(255, 255, 255, 0.2)',  # Set the grid color with transparency
            showgrid=True,  # Show the grid lines
        ),
        yaxis=dict(
            gridcolor='rgba(255, 255, 255, 0.2)',  # Set the grid color with transparency for the y-axis
            showgrid=True,  # Show the grid lines for the y-axis
        ),
        plot_bgcolor='rgba(0, 0, 0, 0)',  # Set the plot area background to transparent
        paper_bgcolor='rgba(0, 0, 0, 0)',  # Set the entire chart background to transparent
        font=dict(color='white'),  # Set the font color for the labels and legend to white
        title=dict(text=f'{title_coin_symbol}',  # Set the title of the chart
                   x=0.5,  # Set the title position to the center of the chart
                   y=0.95,  # Set the title position relative to the y-axis
                   font=dict(size=24)  # Set the font size of the title
                   ),
        xaxis_rangeslider_visible=False
    )

    # Convert the chart to JSON to pass it to the template
    chart_json = fig.to_json()

    # Get coin data from the session
    coin_data = request.session.get('coin_data', [])
    coin_info = {}
    for coin in coin_data:
        real_coin_symbol = coin_symbol.replace('USDT', '').lower()
        if coin['symbol'] == real_coin_symbol:
            coin_info = {
                'name': coin.get('name'),
                'symbol': real_coin_symbol,
                'current_price': coin.get('current_price'),
                'daily_price_change': coin.get('price_change_percentage_24h'),
                'daily_volume': coin.get('total_volume'),
                'market_cap': coin.get('market_cap'),
            }
            break

    # Pass the selected coin's data and chart JSON to the template
    return render(request, 'crypto_pulse/coin_chart.html', {'chart_json': chart_json,
                                                            'coin_info': coin_info,
                                                            'bids': bids_data,
                                                            'asks': asks_data})

def coin_details(request, coin_name):
    # Fetch data for description
    if coin_name.lower() == 'xrp':
        coin_name = 'ripple'
    if coin_name.lower() == 'binance usd':
        coin_name = 'binance-usd'
    if coin_name.lower() == 'bnb':
        coin_name = 'busd'
    if coin_name.lower() == 'usd coin' or coin_name.lower() == 'usd%20coin':
        coin_name = 'usd-coin'
    if coin_name.lower() == 'wrapped bitcoin' or coin_name.lower() == 'wrapped%20bitcoin':
        coin_name = 'wrapped-bitcoin'
    if coin_name.lower() == 'shiba inu' or coin_name.lower() == 'shiba%20inu':
        coin_name = 'shiba-inu'
    if coin_name.lower() == 'bitcoin cash' or coin_name.lower() == 'bitcoin%20cash':
        coin_name = 'bitcoin-cash'
    if coin_name.lower() == 'ethereum classic' or coin_name.lower() == 'ethereum%20classic':
        coin_name = 'ethereum-classic'
    if coin_name.lower() == 'cosmos hub' or coin_name.lower() == 'cosmos%20hub':
        coin_name = 'cosmos-hub'

    coin_details = fetch_coin_details(API_BASE_URL_DETAILS, coin_name=coin_name)

    if coin_name.lower() == 'ripple':
        coin_name = 'xrp'

    coin_history = fetch_coin_history(API_COINCAP_URL, coin_name=coin_name)

    if coin_history is None:
        messages.error(request, 'Failed to fetch data. Please try again.')
        return render(request, 'crypto_pulse/coin_chart.html')

    data = coin_history['data']

    # Parse coin history
    data = coin_history['data']

    prices = []
    dates = []

    for data_point in data:
        price = data_point['priceUsd']
        date = data_point['date']

        prices.append(float(price))
        dates.append(date)

    # Convert formatted_dates to datetime objects
    formatted_dates = [parse_date(date_string) for date_string in dates]

    # Clear plots
    plt.clf()
    plt.figure(figsize=(11, 4))  # Adjust the height to 4

    # Creating line chart
    plt.plot(formatted_dates, prices, linewidth=2)

    # Plot info and style
    plt.style.use('_mpl-gallery')
    plt.xlabel('', color='white')  # Increase font size to 12
    plt.ylabel('', color='white')  # Increase font size to 12
    plt.title(f'{coin_name} price', color='white')  # Increase font size to 16
    plt.tight_layout()

    # Remove the grid from both axes
    plt.grid(False)

    # Style tick labels on both axes to have a white font color and increase font size to 10
    plt.tick_params(axis='x', colors='white')
    plt.tick_params(axis='y', colors='white', labelsize=8)

    # Configure x-axis tick marks to display fewer dates
    locator = mdates.AutoDateLocator(minticks=5, maxticks=9)
    formatter = mdates.ConciseDateFormatter(locator)
    plt.gca().xaxis.set_major_locator(locator)
    plt.gca().xaxis.set_major_formatter(formatter)

    # Convert the plot to an image
    buffer = BytesIO()
    plt.savefig(buffer, format='png', transparent=True)
    buffer.seek(0)
    line_chart = base64.b64encode(buffer.getvalue()).decode('utf-8')
    buffer.close()

    # Check if data was successfully fetched
    if coin_details is None:
        messages.error(request, 'Failed to fetch data. Please try again.')
        return render(request, 'crypto_pulse/coin_chart.html')

    # Create a dictionary to store descriptions
    description = coin_details['description']['en']
    name = coin_details['name']
    symbol = coin_details['symbol'].upper()

    # Clean the description by removing URLs and links
    cleaned_description = description.replace('<a href="', '')
    url_pattern = re.compile(r'http[s]?://\S+|www\.\S+')
    cleaned_description = re.sub(url_pattern, '', cleaned_description)

    context = {
        'description': cleaned_description,
        'name': name,
        'symbol': symbol,
        'line_chart': line_chart
    }

    return render(request, 'crypto_pulse/coin_details.html', context)


import pandas as pd
import numpy
import os
import sys
import csv
import logging
import psycopg2

from psycopg2.extensions import register_adapter, AsIs
from cloudharness import applications

def addapt_numpy_int64(numpy_int64):
    return AsIs(numpy_int64)

register_adapter(numpy.int64, addapt_numpy_int64)

app = applications.get_configuration("pub-chem-index")
conn_string = f"postgres://{app.db_name}:{app.harness.database.postgres.ports[0]['port']}/asu?user={app.harness.database.user}&password={app.harness.database.get('pass')}"

logging.basicConfig(stream=sys.stdout, level=logging.DEBUG)

NUM_PROCESSES = 2
NUM_QUEUE_ITEMS = 20

#
# WARNING!!! use head command on files for debugging
# head -n 500000 CID-SMILES > CID-SMILES-head
#

# "https://ftp.ncbi.nlm.nih.gov/pubchem/Compound/Extras/CID-Synonym-unfiltered.gz"

added_col_dic = {
    "CID-InChI-Key": ["CID", "InChI", "Key"],
    "CID-Mass": ["CID", "Molecule", "Mass1", "Mass2"],
    "CID-PMID": ["CID", "PMID"],
    "CID-Parent": ["CID", "Parent"],
    "CID-Patent": ["CID", "Patent"],
    "CID-SID": ["CID", "SID"],
    "CID-MeSH": ["CID", "MeSH"],
    "CID-SMILES": ["CID", "SMILES"],
    "CID-Synonym-filtered": ["CID", "Synonym"],
    "CID-Synonym-unfiltered": ["CID", "Synonym"],
    "CID-Title": ["CID", "Title"],
    "CID-IUPAC": ["CID", "IUPAC"],
    "CID-Component": ["CID", "component"],
}

gin_indexes_tables = ['CID-Title', 'CID-MeSH', 'CID-IUPAC', 'CID-InChI-Key', 'CID-Synonym-filtered']

def change_permissions_recursive(path, mode):
    for root, dirs, files in os.walk(path, topdown=False):
        for file in [os.path.join(root, f) for f in files]:
            if file.startswith("export-"):
                os.chmod(file, mode)
    # os.chmod(path, mode)

def execute_sql(conn, command):
    with conn.cursor() as cur:
        logging.info(f"Execute {command}")
        cur.execute(command)
        conn.commit()

def create_table(conn, table_name):
    # Populate GIN indexed table, this will take about 30 minutes.
    column_names = added_col_dic[table_name]
    column_names = [x.upper() for x in column_names]
    table_name = table_name.replace("-", "_").upper()

    str_column_names = ""
    for i in column_names:
        if i == "CID":
            str_column_names += i + " INTEGER NOT NULL,"
        else:
            str_column_names += i + " VARCHAR,"

    str_column_names = str_column_names[: len(str_column_names) - 1]

    sql_drop_table = f"DROP TABLE IF EXISTS {table_name}"
    sql_create_table = f"CREATE TABLE {table_name} ({str_column_names})"

    execute_sql(conn, sql_drop_table)
    execute_sql(conn, sql_create_table)

    logging.info("Table created %s ", table_name)

def bulk_insert(conn, data, file_name):
    with conn.cursor() as cur:
        table_name = file_name.replace("-", "_").upper()
        columns = added_col_dic[file_name]
        column_list = ", ".join(columns)
        records_list_template = ",".join(["%s"] * len(data))
        insert_query = "insert into {table_name} ({columns}) values {};".format(
            records_list_template, table_name=table_name, columns=column_list
        )
        cur.execute(insert_query, data)
        conn.commit()

def create_indexes(conn, table_name, create_gin):
    column_names = added_col_dic[table_name]
    column_names = [x.upper() for x in column_names]
    main_column = column_names[1].lower()
    table_name = table_name.replace("-", "_").lower()

    if create_gin:
        logging.info("Start creating indexes")
        execute_sql(conn, "CREATE EXTENSION IF NOT EXISTS pg_trgm;")
        execute_sql(conn, f"CREATE INDEX IF NOT EXISTS idx_gin_{table_name} ON {table_name} USING gin ({main_column} gin_trgm_ops);")

    execute_sql(conn, f"CREATE INDEX IF NOT EXISTS cid_idx_{table_name} ON {table_name} (CID);")
    logging.info("Finish creating indexes")

def get_line(file_name):
    with open(file_name) as file:
        for i in file:
            yield i

def go():
    logging.info(f"Connecting with string: {conn_string}")
    conn = psycopg2.connect(conn_string)

    file = sys.argv[1]
    file_name = os.path.basename(file)
    logging.info(f"Populating table using file {file_name}")

    column_name = ["CID", file_name]
    types = {file_name: "string", "CID": "Int64"}
    column_names = added_col_dic[file_name]
    # column_names = [x.upper() for x in column_names]
    main_column = column_names[1]  # .upper()
    gin_indexed = file_name in gin_indexes_tables

    if file_name in added_col_dic:
        column_name = added_col_dic[file_name]
        types = {"CID": "string"}
        for c in column_name:
            if c != "CID":
                types[c] = "string"

    create_table(conn, file_name)

    encoding = "UTF-8"
    if file_name == "CID-Title":
        encoding = "Latin"
    chunksize = 200000
    column_slice_size = len(column_names)

    logging.info("Inserting...")
    record_counter = 0
    with open(file, "rb") as f:
        data = []
        for line in f:
            data.append(tuple(line.decode(encoding).replace("\n","").split("\t")[:column_slice_size]))
            if len(data) == chunksize:
                with conn:
                    bulk_insert(conn, data, file_name)
                    record_counter += chunksize
                    if record_counter%10000 == 0:
                        logging.info(f"Total number of records inserted: {record_counter}")
                data = []

    # insert the left over data (if there is any)
    if len(data) != chunksize:
        with conn:
            bulk_insert(conn, data, file_name)
            record_counter += len(data)
            logging.info(f"Total number of records inserted: {record_counter}")
    logging.info("Inserting done...")

    create_indexes(conn, file_name, gin_indexed)

go()


# -*- coding: utf-8 -*-
"""
Jianyu Chen

This is a project for optimal control.
Python code for 1-dim double well system
"""
#%%
import torch
from torch import nn
from torch.nn import functional as F
import math
import numpy as np
import matplotlib.pyplot as plt
#%%
t = np.arange(100) * 0.01
print(t)
sigma = 1
#%%
u= np.random.normal(0, 0.008, 100)
#u = np.linspace(0, 1, 100)
print(u)
sigma = 1
def runge_kutta(x, t, dt, f, u, i):
    """   x is the initial value for x
          t is the initial value for t
          dt is the time step in t
          f is derivative of function x(t)

    """

    k1 = dt * (f(x, t)+u[i])
    k2 = dt * (f(x + k1, t + dt)+u[i+1])
    return x + (k1 + k2) / 2.

def get_x(u):
    t = 0.
    x = -1.
    T = 1.0
    dt = 0.01
    xs, ts = [x], [0.0]
    def func(x, t):
        return x-x**3
    i = 0
    while i < 99:
        #print (i, t)
        x = runge_kutta(x, t, dt, func, u, i)
        i += 1
        t += dt
        xs.append(x)
        ts.append(t)
    return np.array([np.float32(xx) for xx in xs])
#%%
import math
import numpy as np
import matplotlib.pyplot as plt
def runge_kutta(p, t, dt, f, xs, i):
    """   p is the terminal value for p
          t is the initial value for t
          dt is the time step in t
          f is derivative of function p(t)

    """

    k1 = -dt * (f(p,t)*(3*xs[i]**2-1)-3*xs[i])
    k2 = -dt * (f(p + k1, t + dt)*(3*xs[i-1]**2-1)-3*xs[i-1])
    return p + (k1 + k2) / 2.

def get_p(x):
    t = 1.0
    p = -2*(x[99]-1)/((x[99]-1)**2+1)**2
    dt = 0.01
    ps, ts = [p], [0.0]
    def func(p, t):
        return p
    i = 99
    while i > 0:
        #print (i, t)
        p = runge_kutta(p, t, dt, func, x, i)
        i -= 1
        t -= dt
        ps.append(p)
        ts.append(t)

    return np.array([np.float32(pp) for pp in ps])
#%%
class MLP(nn.Module):
     def __init__(self):#call the constractor class"Module" to perform necessary initialization.
        super().__init__()
        self.hidden1 = nn.Linear(1, 8)#hidden
        self.hidden2 = nn.Linear(8, 16)#hidden
        self.out = nn.Linear(16,1)#out put layer
 #define the forward propagation of the model,how to return the required model output
 #based on the input "t"（input t is "u"）
     def forward(self, u):
        u = self.hidden1(u)
        u = F.relu(u)
        u = self.hidden2(u)
        u = F.relu(u)
        out = self.out(u)
        return out
net = MLP()
#%%
t = torch.tensor(t)
t = t.to(torch.float32)
x = get_x(u)
p = get_p(x)
reversed_p = []
for i in reversed(p):
    reversed_p.append(i)

print(reversed_p)
p = reversed_p
x = torch.tensor(x).to(torch.float32)
p = torch.tensor(p).to(torch.float32)
#%%
print(x)
print(p)
#%%
import numpy as np
import matplotlib.pylab as plt
#from gradient import numerical_gradient
Loss = []
lr =0.05
epoch = 1000
N = 20#iteration times
Q = 2*(x[99]-1)**2
print(Q)
dt = 0.01
dx= (x[1:]-x[0:-1])/dt
dp = (p[1:]-p[0:-1])/dt
rho = 1 #修正系数
updater = torch.optim.SGD(net.parameters(), lr)  # change to adaptive
print(dx.shape)
#%%
def train():

    for ii in range(epoch):
        H = 0
        for j in range(99):
            t_j = t[j].reshape(1)
            u_hat_j = net(torch.tensor(t_j))
            x_j = x[j].reshape(1)
            p_j = p[j].reshape(1)
            H = H + p_j* (x_j-x_j**3+sigma*u_hat_j) - 0.5*(u_hat_j**2+1-3*x_j**2)-0.5*rho*(abs(dx[j]-x_j+x_j**3-sigma*u_hat_j)**2)+abs((dp[j]+p_j*(1-3*x_j**2)+3*x_j)**2)

        l = -(H-Q)/100 # l>0

        updater.zero_grad()
        l.backward()
        updater.step()
        Loss.append(l.detach().item())

    return
#%%
for k in range(N):
    print(k)
    U = [net(torch.tensor(t_j.reshape(1))) for t_j in t]
    u_op = [u.detach().numpy() for u in U]
    #print(u_op)
    flatten_u_op = [element for sublist in u_op for element in sublist]
    print(flatten_u_op)
    x = get_x(flatten_u_op)
    p = get_p(x)

    reversed_p = []
    for i in reversed(p):
        reversed_p.append(i)

    p = reversed_p

    x = torch.tensor(x).to(torch.float32)
    p = torch.tensor(p).to(torch.float32)
    train()
    print(x)
#%%
x = t
y = flatten_u_op
#y = list(map(lambda x:x.detach().numpy(), y))
plt.title("optimal control")
plt.xlabel("Time")
plt.ylabel("u")

plt.plot(x, y)
plt.show()
#%%
x = get_x(y)
print(x)
#%%
import scipy.io as scio
scio.savemat('pathway_data.mat',{'x':x})


# FLASK: location API functions

from flask import Flask, jsonify, request
import pymysql

from __main__ import app
from helpers.location import *
from db_connect.db_connection import connect

# Open database connection
db = connect()
cursor = db.cursor()

# New payment or get payments to a tutor
@app.route("/location", methods=['GET','POST'])
def location():

    # Retrieve parameters
    dist = request.args.get('District')
    city = request.args.get('City')
    country = request.args.get('Country')

    # Find locations
    if request.method == 'GET':
        cursor.execute(get_location(dist, city, country))
        return jsonify(cursor.fetchall())

    # Create new location in database
    elif request.method == 'POST':
        pc = request.args.get('PostalCode')
        cursor.execute(new_location(pc, dist, city, country))
        db.commit()
        return jsonify("Success!")


import uvicorn
from fastapi import FastAPI
from fastapi.middleware.cors import CORSMiddleware
from sockets import sio_app

app = FastAPI()
app.mount('/', app=sio_app)

app.add_middleware(
    CORSMiddleware,
    allow_origins=['*'],
    allow_credentials=True,
    allow_methods=["*"],
    allow_headers=["*"],
)

@app.get('/')
async def home():
    return {'message': 'HelloðŸ‘‹ DevelopersðŸ’»'}

if __name__ == '__main__':
    uvicorn.run('main:app', reload=True)


#!/usr/bin/python3
"""This module defines the class Place that inherits from class BaseModel"""

from models.base_model import BaseModel, Base
from sqlalchemy import Column, String, Integer, Float, ForeignKey
from sqlalchemy.orm import relationship
from sqlalchemy.sql.schema import Table
import os

if os.getenv('HBNB_TYPE_STORAGE') == 'db':
    place_amenity = Table('place_amenity',
                          Base.metadata,
                          Column('place_id',
                                 String(60),
                                 ForeignKey('places.id'),
                                 primary_key=True,
                                 nullable=False),
                          Column('amenity_id',
                                 String(60),
                                 ForeignKey('amenities.id'),
                                 primary_key=True,
                                 nullable=False))

class Place(BaseModel, Base):
    """Defines the methods and attributes of the class"""

    __tablename__ = 'places'

    if os.getenv('HBNB_TYPE_STORAGE') == 'db':
        city_id = Column(String(60), ForeignKey('cities.id'), nullable=False)
        user_id = Column(String(60), ForeignKey('users.id'), nullable=False)
        name = Column(String(128), nullable=False)
        description = Column(String(1024), nullable=False)
        number_rooms = Column(Integer, nullable=False, default=0)
        number_bathrooms = Column(Integer, nullable=False, default=0)
        max_guest = Column(Integer, nullable=False, default=0)
        price_by_night = Column(Integer, nullable=False, default=0)
        latitude = Column(Float, nullable=True)
        longitude = Column(Float, nullable=True)
        reviews = relationship('Review', backref='place',
                               cascade='all, delete, delete-orphan')
        amenities = relationship('Amenity', backref='place_amenities',
                                 secondary=place_amenity, viewonly=False)
    else:
        city_id = ""
        user_id = ""
        name = ""
        description = ""
        number_rooms = 0
        number_bathrooms = 0
        max_guest = 0
        price_by_night = 0
        latitude = 0.0
        longitude = 0.0
        amenity_ids = []

        @property
        def reviews(self):
            """
            Returns the list of Review instances with place_id equals
            to the current place.id.
            """
            from models import storage
            from models.review import Review

            review_of_place = []
            review_dict = storage.all(Review)
            for obj in review_dict.values():
                if obj.place_id == self.id:
                    review_of_place.append(obj)
            return review_of_place

        @property
        def amenities(self):
            """
            Returns the list of Amenity instances based on the attribute
            amenity_ids that contains all Amenity.id linked to the place
            """

            from models import storage
            from models.amenity import Amenity

            amenities_of_place = []
            amenity_dict = storage.all(Amenity)

            for obj in amenity_dict.values():
                if obj.id in self.amenity_ids:
                    amenities_of_place.append(obj)
            return amenities_of_place

        @amenities.setter
        def amenities(self, obj):
            """Append amenity 'id' to the list of amenity ids"""

            if type(obj) != Amenity:
                return

            if obj.id not in self.amenity_ids:
                self.amenity_ids.append(obj.id)

    def __init__(self, *args, **kwargs):
        """Initializes the attributes of the class"""

        super().__init__(*args, **kwargs)


import pathlib
import typing

import sqlalchemy.orm
from fastapi import Depends

import mlrun.common.schemas.model_monitoring
import mlrun.common.schemas.model_monitoring.constants as mm_constants
import mlrun.model_monitoring.stream_processing
import mlrun.model_monitoring.tracking_policy
import server.api.api.endpoints.functions
import server.api.api.utils
import server.api.crud.model_monitoring.helpers
import server.api.utils.scheduler
import server.api.utils.singletons.k8s
from mlrun import feature_store as fstore
from mlrun.model_monitoring.writer import ModelMonitoringWriter
from mlrun.utils import logger
from server.api.api import deps
from server.api.crud.model_monitoring.helpers import Seconds, seconds2minutes

_MODEL_MONITORING_COMMON_PATH = (
    pathlib.Path(__file__).parents[4] / "mlrun" / "model_monitoring"
)
_STREAM_PROCESSING_FUNCTION_PATH = (
    _MODEL_MONITORING_COMMON_PATH / "stream_processing.py"
)
_MONITORING_ORIGINAL_BATCH_FUNCTION_PATH = _MODEL_MONITORING_COMMON_PATH / "batch.py"

_MONITORING_APPLICATION_CONTROLLER_FUNCTION_PATH = (
    _MODEL_MONITORING_COMMON_PATH / "controller_handler.py"
)

_MONITORING_WRITER_FUNCTION_PATH = _MODEL_MONITORING_COMMON_PATH / "writer.py"

class MonitoringDeployment:
    def __init__(
        self,
        parquet_batching_max_events: int = mlrun.mlconf.model_endpoint_monitoring.parquet_batching_max_events,
        max_parquet_save_interval: int = mlrun.mlconf.model_endpoint_monitoring.parquet_batching_timeout_secs,
    ) -> None:
        self._parquet_batching_max_events = parquet_batching_max_events
        self._max_parquet_save_interval = max_parquet_save_interval
        """
        Initialize a MonitoringDeployment object, which handles the deployment & scheduling of:
         1. model monitoring stream
         2. model monitoring batch
         3. model monitoring batch application
         4. model monitoring writer

        :param parquet_batching_max_events: Maximum number of events that will be used for writing the monitoring
                                            parquet by the monitoring stream function.
        :param max_parquet_save_interval:   Maximum number of seconds to hold events before they are written to the
                                            monitoring parquet target. Note that this value will be used to handle the
                                            offset by the scheduled batch job.
        """

    def deploy_monitoring_functions(
        self,
        project: str,
        model_monitoring_access_key: str,
        db_session: sqlalchemy.orm.Session,
        auth_info: mlrun.common.schemas.AuthInfo,
        tracking_policy: mlrun.model_monitoring.tracking_policy.TrackingPolicy,
    ):
        """
        Invoking monitoring deploying functions.

        :param project:                     The name of the project.
        :param model_monitoring_access_key: Access key to apply the model monitoring process.
        :param db_session:                  A session that manages the current dialog with the database.
        :param auth_info:                   The auth info of the request.
        :param tracking_policy:             Model monitoring configurations.
        """
        self.deploy_model_monitoring_stream_processing(
            project=project,
            model_monitoring_access_key=model_monitoring_access_key,
            db_session=db_session,
            auth_info=auth_info,
            tracking_policy=tracking_policy,
        )
        self.deploy_model_monitoring_batch_processing(
            project=project,
            model_monitoring_access_key=model_monitoring_access_key,
            db_session=db_session,
            auth_info=auth_info,
            tracking_policy=tracking_policy,
            tracking_offset=Seconds(self._max_parquet_save_interval),
            function_name=mm_constants.MonitoringFunctionNames.BATCH,
        )

    def deploy_model_monitoring_stream_processing(
        self,
        project: str,
        model_monitoring_access_key: str,
        db_session: sqlalchemy.orm.Session,
        auth_info: mlrun.common.schemas.AuthInfo,
        tracking_policy: mlrun.model_monitoring.tracking_policy.TrackingPolicy,
    ) -> None:
        """
        Deploying model monitoring stream real time nuclio function. The goal of this real time function is
        to monitor the log of the data stream. It is triggered when a new log entry is detected.
        It processes the new events into statistics that are then written to statistics databases.

        :param project:                     The name of the project.
        :param model_monitoring_access_key: Access key to apply the model monitoring process.
        :param db_session:                  A session that manages the current dialog with the database.
        :param auth_info:                   The auth info of the request.
        :param tracking_policy:             Model monitoring configurations.
        """

        logger.info(
            "Checking if model monitoring stream is already deployed",
            project=project,
        )
        try:
            # validate that the model monitoring stream has not yet been deployed
            mlrun.runtimes.function.get_nuclio_deploy_status(
                name="model-monitoring-stream",
                project=project,
                tag="",
                auth_info=auth_info,
            )
            logger.info(
                "Detected model monitoring stream processing function already deployed",
                project=project,
            )
            return
        except mlrun.errors.MLRunNotFoundError:
            logger.info(
                "Deploying model monitoring stream processing function", project=project
            )

        # Get parquet target value for model monitoring stream function
        parquet_target = (
            server.api.crud.model_monitoring.helpers.get_monitoring_parquet_path(
                db_session=db_session, project=project
            )
        )

        fn = self._initial_model_monitoring_stream_processing_function(
            project=project,
            model_monitoring_access_key=model_monitoring_access_key,
            tracking_policy=tracking_policy,
            auth_info=auth_info,
            parquet_target=parquet_target,
        )

        # Adding label to the function - will be used to identify the stream pod
        fn.metadata.labels = {"type": "model-monitoring-stream"}

        server.api.api.endpoints.functions._build_function(
            db_session=db_session,
            auth_info=auth_info,
            function=fn,
        )

    def deploy_model_monitoring_controller(
        self,
        project: str,
        model_monitoring_access_key: str,
        db_session: sqlalchemy.orm.Session,
        auth_info: mlrun.common.schemas.AuthInfo,
        tracking_policy: mlrun.model_monitoring.tracking_policy.TrackingPolicy,
    ) -> typing.Union[mlrun.runtimes.kubejob.KubejobRuntime, None]:
        """
        Submit model monitoring application controller job along with deploying the model monitoring writer function.
        While the main goal of the controller job is to handle the monitoring processing and triggering applications,
        the goal of the model monitoring writer function is to write all the monitoring application results to the
        databases. Note that the default scheduling policy of the controller job is to run every 5 min.

        :param project:                     The name of the project.
        :param model_monitoring_access_key: Access key to apply the model monitoring process.
        :param db_session:                  A session that manages the current dialog with the database.
        :param auth_info:                   The auth info of the request.
        :param tracking_policy:             Model monitoring configurations, including the required controller
                                            configurations such as the base period (5 minutes by default) and
                                            the default controller image (`mlrun/mlrun` by default).

        :return: Model monitoring controller job as a runtime function.
        """

        self.deploy_model_monitoring_writer_application(
            project=project,
            model_monitoring_access_key=model_monitoring_access_key,
            db_session=db_session,
            auth_info=auth_info,
            tracking_policy=tracking_policy,
        )

        return self.deploy_model_monitoring_batch_processing(
            project=project,
            model_monitoring_access_key=model_monitoring_access_key,
            db_session=db_session,
            auth_info=auth_info,
            tracking_policy=tracking_policy,
            function_name=mm_constants.MonitoringFunctionNames.APPLICATION_CONTROLLER,
        )

    def deploy_model_monitoring_batch_processing(
        self,
        project: str,
        model_monitoring_access_key: str,
        db_session: sqlalchemy.orm.Session,
        auth_info: mlrun.common.schemas.AuthInfo,
        tracking_policy: mlrun.model_monitoring.tracking_policy.TrackingPolicy,
        with_schedule: bool = True,
        overwrite: bool = False,
        tracking_offset: Seconds = Seconds(0),
        function_name: str = mm_constants.MonitoringFunctionNames.BATCH,
    ) -> typing.Union[mlrun.runtimes.kubejob.KubejobRuntime, None]:
        """
        Deploying model monitoring batch job.
        The goal of this job is to identify drift in the data based on the latest batch of events. By default,
        this job is executed on the hour every hour.
        Note that if this job was already deployed then you will either have to pass overwrite=True or
        to delete the old job before deploying a new one.

        :param project:                     The name of the project.
        :param model_monitoring_access_key: Access key to apply the model monitoring process.
        :param db_session:                  A session that manages the current dialog with the database.
        :param auth_info:                   The auth info of the request.
        :param tracking_policy:             Model monitoring configurations.
        :param with_schedule:               If true, submit a scheduled batch drift job.
        :param overwrite:                   If true, overwrite the existing model monitoring batch job.
        :param tracking_offset:             Offset for the tracking policy (for synchronization with the stream)
        :param function_name:               model-monitoring-batch or model-monitoring-controller
                                            indicates witch one to deploy.

        :return: Model monitoring batch job as a runtime function.
        """
        job_valid_names = [
            mm_constants.MonitoringFunctionNames.BATCH,
            mm_constants.MonitoringFunctionNames.APPLICATION_CONTROLLER,
        ]
        if function_name not in job_valid_names:
            raise mlrun.errors.MLRunRuntimeError(
                f"Model Monitoring batch job can be only within {job_valid_names}"
            )
        fn = None
        if not overwrite:
            logger.info(
                f"Checking if {function_name.replace('-',' ')} processing function is already deployed",
                project=project,
            )

            # Try to list functions that named model monitoring batch
            # to make sure that this job has not yet been deployed
            try:
                fn = server.api.crud.Functions().get_function(
                    db_session=db_session,
                    name=function_name,
                    project=project,
                )
                logger.info(
                    f"Detected {function_name.replace('-',' ')} processing function already deployed",
                    project=project,
                )

            except mlrun.errors.MLRunNotFoundError:
                logger.info(
                    f"Deploying {function_name.replace('-',' ')} processing function ",
                    project=project,
                )

        if not fn:
            # Create a monitoring batch job function object
            fn = self._get_model_monitoring_batch_function(
                project=project,
                model_monitoring_access_key=model_monitoring_access_key,
                db_session=db_session,
                auth_info=auth_info,
                image=tracking_policy.default_batch_image
                if function_name == mm_constants.MonitoringFunctionNames.BATCH
                else tracking_policy.default_controller_image,
                function_name=function_name,
            )

            # Get the function uri
            function_uri = fn.save(versioned=True)

            if with_schedule:
                if not overwrite:
                    try:
                        server.api.utils.scheduler.Scheduler().get_schedule(
                            db_session=db_session,
                            project=project,
                            name=function_name,
                        )
                        logger.info(
                            f"Already deployed {function_name.replace('-',' ')} scheduled job function ",
                            project=project,
                        )
                        return
                    except mlrun.errors.MLRunNotFoundError:
                        logger.info(
                            f"Deploying {function_name.replace('-',' ')} scheduled job function ",
                            project=project,
                        )
                # Submit batch scheduled job
                self._submit_schedule_batch_job(
                    project=project,
                    function_uri=function_uri,
                    db_session=db_session,
                    auth_info=auth_info,
                    tracking_policy=tracking_policy,
                    tracking_offset=tracking_offset,
                    function_name=function_name,
                )
        return fn

    def deploy_model_monitoring_writer_application(
        self,
        project,
        model_monitoring_access_key,
        db_session,
        auth_info,
        tracking_policy,
    ):
        """
        Deploying model monitoring writer real time nuclio function. The goal of this real time function is
        to write all the monitoring application result to the databases. It is triggered by those applications.
        It processes and writes the result to the databases.

        :param project:                     The name of the project.
        :param model_monitoring_access_key: Access key to apply the model monitoring process.
        :param db_session:                  A session that manages the current dialog with the database.
        :param auth_info:                   The auth info of the request.
        :param tracking_policy:             Model monitoring configurations.
        """

        logger.info(
            "Checking if model monitoring writer is already deployed",
            project=project,
        )
        try:
            # validate that the model monitoring stream has not yet been deployed
            mlrun.runtimes.function.get_nuclio_deploy_status(
                name=mm_constants.MonitoringFunctionNames.WRITER,
                project=project,
                tag="",
                auth_info=auth_info,
            )
            logger.info(
                "Detected model monitoring writer processing function already deployed",
                project=project,
            )
            return
        except mlrun.errors.MLRunNotFoundError:
            logger.info(
                "Deploying model monitoring writer processing function", project=project
            )

        fn = self._initial_model_monitoring_writer_function(
            project=project,
            model_monitoring_access_key=model_monitoring_access_key,
            tracking_policy=tracking_policy,
            auth_info=auth_info,
        )

        # Adding label to the function - will be used to identify the stream pod
        fn.metadata.labels = {"type": "model-monitoring-writer"}

        server.api.api.endpoints.functions._build_function(
            db_session=db_session,
            auth_info=auth_info,
            function=fn,
        )

    def _initial_model_monitoring_stream_processing_function(
        self,
        project: str,
        model_monitoring_access_key: str,
        tracking_policy: mlrun.model_monitoring.tracking_policy.TrackingPolicy,
        auth_info: mlrun.common.schemas.AuthInfo,
        parquet_target: str,
    ):
        """
        Initialize model monitoring stream processing function.

        :param project:                     Project name.
        :param model_monitoring_access_key: Access key to apply the model monitoring process. Please note that in CE
                                            deployments this parameter will be None.
        :param tracking_policy:             Model monitoring configurations.
        :param auth_info:                   The auth info of the request.
        :param parquet_target:              Path to model monitoring parquet file that will be generated by the
                                            monitoring stream nuclio function.

        :return:                            A function object from a mlrun runtime class

        """

        # Initialize Stream Processor object
        stream_processor = (
            mlrun.model_monitoring.stream_processing.EventStreamProcessor(
                project=project,
                parquet_batching_max_events=self._parquet_batching_max_events,
                parquet_batching_timeout_secs=self._max_parquet_save_interval,
                parquet_target=parquet_target,
                model_monitoring_access_key=model_monitoring_access_key,
            )
        )

        # Create a new serving function for the streaming process
        function = mlrun.code_to_function(
            name="model-monitoring-stream",
            project=project,
            filename=str(_STREAM_PROCESSING_FUNCTION_PATH),
            kind="serving",
            image=tracking_policy.stream_image,
        )

        # Create monitoring serving graph
        stream_processor.apply_monitoring_serving_graph(function)

        # Set the project to the serving function
        function.metadata.project = project

        # Add stream triggers
        function = self._apply_stream_trigger(
            project=project,
            function=function,
            model_monitoring_access_key=model_monitoring_access_key,
            auth_info=auth_info,
        )

        # Apply feature store run configurations on the serving function
        run_config = fstore.RunConfig(function=function, local=False)
        function.spec.parameters = run_config.parameters

        return function

    def _get_model_monitoring_batch_function(
        self,
        project: str,
        model_monitoring_access_key: str,
        db_session: sqlalchemy.orm.Session,
        auth_info: mlrun.common.schemas.AuthInfo,
        image: str,
        function_name: str = "model-monitoring-batch",
    ):
        """
        Initialize model monitoring batch function.

        :param project:                     project name.
        :param model_monitoring_access_key: access key to apply the model monitoring process. Please note that in CE
                                            deployments this parameter will be None.
        :param db_session:                  A session that manages the current dialog with the database.
        :param auth_info:                   The auth info of the request.
        :param image:                       Base docker image to use for building the function container
        :param function_name:               model-monitoring-batch or model-monitoring-controller
                                            indicates witch one to create.
        :return:                            A function object from a mlrun runtime class

        """
        filename = (
            str(_MONITORING_ORIGINAL_BATCH_FUNCTION_PATH)
            if function_name == "model-monitoring-batch"
            else str(_MONITORING_APPLICATION_CONTROLLER_FUNCTION_PATH)
        )
        # Create job function runtime for the model monitoring batch
        function: mlrun.runtimes.KubejobRuntime = mlrun.code_to_function(
            name=function_name,
            project=project,
            filename=filename,
            kind="job",
            image=image,
            handler="handler",
        )
        function.set_db_connection(server.api.api.utils.get_run_db_instance(db_session))

        # Set the project to the job function
        function.metadata.project = project

        if not mlrun.mlconf.is_ce_mode():
            function = self._apply_access_key_and_mount_function(
                project=project,
                function=function,
                model_monitoring_access_key=model_monitoring_access_key,
                auth_info=auth_info,
                function_name=function_name,
            )

        # Enrich runtime with the required configurations
        server.api.api.utils.apply_enrichment_and_validation_on_function(
            function, auth_info
        )

        return function

    @classmethod
    def _submit_schedule_batch_job(
        cls,
        project: str,
        function_uri: str,
        db_session: sqlalchemy.orm.Session,
        auth_info: mlrun.common.schemas.AuthInfo,
        tracking_policy: mlrun.model_monitoring.tracking_policy.TrackingPolicy,
        tracking_offset: Seconds = Seconds(0),
        function_name: str = "model-monitoring-batch",
    ):
        """
        Create a new scheduled monitoring batch job analysis based on the model-monitoring-batch function that has
        been already registered.

        :param project:         Project name.
        :param function_uri:    Function URI of the registered model monitoring batch job. This URI includes the
                                related project name, function name, and hash key.
        :param db_session:      A session that manages the current dialog with the database.
        :param auth_info:       The auth info of the request.
        :param tracking_policy: Model monitoring configurations.
        :param tracking_offset: Offset for the tracking policy (for synchronization with the stream).

        """

        function_uri = function_uri.replace("db://", "")

        task = mlrun.new_task(name=function_name, project=project)
        task.spec.function = function_uri

        schedule, batch_dict = cls._generate_schedule_and_interval_dict(
            function_name=function_name,
            tracking_policy=tracking_policy,
            tracking_offset=tracking_offset,
        )

        task.spec.parameters[
            mlrun.common.schemas.model_monitoring.EventFieldType.BATCH_INTERVALS_DICT
        ] = batch_dict

        data = {
            "task": task.to_dict(),
            "schedule": schedule,
        }

        logger.info(
            f"Deploying {function_name.replace('-',' ')} processing function",
            project=project,
        )

        # Add job schedule policy (every hour by default)
        server.api.api.utils.submit_run_sync(
            db_session=db_session, auth_info=auth_info, data=data
        )

    @classmethod
    def _generate_schedule_and_interval_dict(
        cls,
        function_name: str,
        tracking_policy: mlrun.model_monitoring.tracking_policy.TrackingPolicy,
        tracking_offset: Seconds,
    ) -> typing.Tuple[str, typing.Dict[str, int]]:
        """Generate schedule cron string along with the batch interval dictionary according to the providing
        function name. As for the model monitoring controller function, the dictionary batch interval is
        corresponding to the scheduling policy.

        :param tracking_policy: Model monitoring configurations.
        :param tracking_offset: Offset for the tracking policy (for synchronization with the stream).

        :return: A tuple of:
         [0] = Schedule cron string
         [1] = Dictionary of the batch interval.
        """

        if function_name == mm_constants.MonitoringFunctionNames.BATCH:
            # Apply batching interval params
            interval_list = [
                tracking_policy.default_batch_intervals.minute,
                tracking_policy.default_batch_intervals.hour,
                tracking_policy.default_batch_intervals.day,
            ]
            (
                minutes,
                hours,
                days,
            ) = server.api.crud.model_monitoring.helpers.get_batching_interval_param(
                interval_list
            )
            schedule = server.api.crud.model_monitoring.helpers.convert_to_cron_string(
                tracking_policy.default_batch_intervals,
                minute_delay=seconds2minutes(tracking_offset),
            )
        else:
            # Apply monitoring controller params
            minutes = tracking_policy.base_period
            hours = days = 0
            schedule = f"*/{tracking_policy.base_period} * * * *"
        batch_dict = {
            mm_constants.EventFieldType.MINUTES: minutes,
            mm_constants.EventFieldType.HOURS: hours,
            mm_constants.EventFieldType.DAYS: days,
        }
        return schedule, batch_dict

    def _apply_stream_trigger(
        self,
        project: str,
        function: mlrun.runtimes.ServingRuntime,
        model_monitoring_access_key: str = None,
        auth_info: mlrun.common.schemas.AuthInfo = Depends(deps.authenticate_request),
        function_name: str = None,
    ) -> mlrun.runtimes.ServingRuntime:
        """Adding stream source for the nuclio serving function. By default, the function has HTTP stream trigger along
        with another supported stream source that can be either Kafka or V3IO, depends on the stream path schema that is
        defined under mlrun.mlconf.model_endpoint_monitoring.store_prefixes. Note that if no valid stream path has been
        provided then the function will have a single HTTP stream source.

        :param project:                     Project name.
        :param function:                    The serving function object that will be applied with the stream trigger.
        :param model_monitoring_access_key: Access key to apply the model monitoring stream function when the stream is
                                            schema is V3IO.
        :param auth_info:                   The auth info of the request.
        :param function_name:             the name of the function that be applied with the stream trigger,
                                            None for model_monitoring_stream

        :return: ServingRuntime object with stream trigger.
        """

        # Get the stream path from the configuration
        # stream_path = mlrun.mlconf.get_file_target_path(project=project, kind="stream", target="stream")
        stream_path = server.api.crud.model_monitoring.get_stream_path(
            project=project, application_name=function_name
        )

        if stream_path.startswith("kafka://"):
            topic, brokers = mlrun.datastore.utils.parse_kafka_url(url=stream_path)
            # Generate Kafka stream source
            stream_source = mlrun.datastore.sources.KafkaSource(
                brokers=brokers,
                topics=[topic],
            )
            function = stream_source.add_nuclio_trigger(function)

        if not mlrun.mlconf.is_ce_mode():
            function = self._apply_access_key_and_mount_function(
                project=project,
                function=function,
                model_monitoring_access_key=model_monitoring_access_key,
                auth_info=auth_info,
                function_name=function_name,
            )
            if stream_path.startswith("v3io://"):
                # Generate V3IO stream trigger
                function.add_v3io_stream_trigger(
                    stream_path=stream_path,
                    name="monitoring_stream_trigger"
                    if function_name is None
                    else f"monitoring_{function_name}_trigger",
                    access_key=model_monitoring_access_key
                    if function_name != mm_constants.MonitoringFunctionNames.STREAM
                    else None,
                )
        # Add the default HTTP source
        http_source = mlrun.datastore.sources.HttpSource()
        function = http_source.add_nuclio_trigger(function)

        return function

    @staticmethod
    def _apply_access_key_and_mount_function(
        project: str,
        function: typing.Union[
            mlrun.runtimes.KubejobRuntime, mlrun.runtimes.ServingRuntime
        ],
        model_monitoring_access_key: str,
        auth_info: mlrun.common.schemas.AuthInfo,
        function_name: str = None,
    ) -> typing.Union[mlrun.runtimes.KubejobRuntime, mlrun.runtimes.ServingRuntime]:
        """Applying model monitoring access key on the provided function when using V3IO path. In addition, this method
        mount the V3IO path for the provided function to configure the access to the system files.

        :param project:                     Project name.
        :param function:                    Model monitoring function object that will be filled with the access key and
                                            the access to the system files.
        :param model_monitoring_access_key: Access key to apply the model monitoring stream function when the stream is
                                            schema is V3IO.
        :param auth_info:                   The auth info of the request.

        :return: function runtime object with access key and access to system files.
        """

        if function_name in mm_constants.MonitoringFunctionNames.all():
            # Set model monitoring access key for managing permissions
            function.set_env_from_secret(
                mlrun.common.schemas.model_monitoring.ProjectSecretKeys.ACCESS_KEY,
                server.api.utils.singletons.k8s.get_k8s_helper().get_project_secret_name(
                    project
                ),
                server.api.crud.secrets.Secrets().generate_client_project_secret_key(
                    server.api.crud.secrets.SecretsClientType.model_monitoring,
                    mlrun.common.schemas.model_monitoring.ProjectSecretKeys.ACCESS_KEY,
                ),
            )

            function.metadata.credentials.access_key = model_monitoring_access_key
            function.apply(mlrun.v3io_cred())

            # Ensure that the auth env vars are set
            server.api.api.utils.ensure_function_has_auth_set(function, auth_info)
        return function

    def _initial_model_monitoring_writer_function(
        self, project, model_monitoring_access_key, tracking_policy, auth_info
    ):
        """
        Initialize model monitoring writer function.

        :param project:                     Project name.
        :param model_monitoring_access_key: Access key to apply the model monitoring process. Please note that in CE
                                            deployments this parameter will be None.
        :param tracking_policy:             Model monitoring configurations.
        :param auth_info:                   The auth info of the request.

        :return:                            A function object from a mlrun runtime class

        """

        # Create a new serving function for the streaming process
        function = mlrun.code_to_function(
            name=mm_constants.MonitoringFunctionNames.WRITER,
            project=project,
            filename=str(_MONITORING_WRITER_FUNCTION_PATH),
            kind="serving",
            image=tracking_policy.default_controller_image,
        )

        # Create writer monitoring serving graph
        graph = function.set_topology("flow")
        graph.to(ModelMonitoringWriter(project=project)).respond()  # writer

        # Set the project to the serving function
        function.metadata.project = project

        # create v3io stream for  model_monitoring_writer | model monitoring application
        server.api.api.endpoints.functions.create_model_monitoring_stream(
            project=project,
            function=function,
            monitoring_application=mm_constants.MonitoringFunctionNames.WRITER,
            stream_path=server.api.crud.model_monitoring.get_stream_path(
                project=project,
                application_name=mm_constants.MonitoringFunctionNames.WRITER,
            ),
            access_key=model_monitoring_access_key,
        )

        # Add stream triggers
        function = self._apply_stream_trigger(
            project=project,
            function=function,
            model_monitoring_access_key=model_monitoring_access_key,
            auth_info=auth_info,
            function_name=mm_constants.MonitoringFunctionNames.WRITER,
        )

        # Apply feature store run configurations on the serving function
        run_config = fstore.RunConfig(function=function, local=False)
        function.spec.parameters = run_config.parameters

        return function

def get_endpoint_features(
    feature_names: typing.List[str],
    feature_stats: dict = None,
    current_stats: dict = None,
) -> typing.List[mlrun.common.schemas.Features]:
    """
    Getting a new list of features that exist in feature_names along with their expected (feature_stats) and
    actual (current_stats) stats. The expected stats were calculated during the creation of the model endpoint,
    usually based on the data from the Model Artifact. The actual stats are based on the results from the latest
    model monitoring batch job.

    param feature_names: List of feature names.
    param feature_stats: Dictionary of feature stats that were stored during the creation of the model endpoint
                         object.
    param current_stats: Dictionary of the latest stats that were stored during the last run of the model monitoring
                         batch job.

    return: List of feature objects. Each feature has a name, weight, expected values, and actual values. More info
            can be found under `mlrun.common.schemas.Features`.
    """

    # Initialize feature and current stats dictionaries
    safe_feature_stats = feature_stats or {}
    safe_current_stats = current_stats or {}

    # Create feature object and add it to a general features list
    features = []
    for name in feature_names:
        if feature_stats is not None and name not in feature_stats:
            logger.warn("Feature missing from 'feature_stats'", name=name)
        if current_stats is not None and name not in current_stats:
            logger.warn("Feature missing from 'current_stats'", name=name)
        f = mlrun.common.schemas.Features.new(
            name, safe_feature_stats.get(name), safe_current_stats.get(name)
        )
        features.append(f)
    return features