from traceback import print_tb
import telebot
from telebot import types
import MySQLdb
from datetime import datetime
from telebot.types import ReplyKeyboardRemove

# DB & Bot connection
db = MySQLdb.connect("localhost", "root", "1", "db_bikes") or die(
    "could not connect to database")
bot = telebot.TeleBot('5304555854:AAEuWKOhEklQASHjkG7oU0F8AV4jwukkk5Q')

# dependencies
show_bike = ''
free_bike = ()
NAME = []

# Start command
@bot.message_handler(commands=['start'])
def start_message(message):
    global NAME
    bot.send_message(message.chat.id, 'Welcome, please Enter your name')
    NAME = []

# Show all bikes
@bot.message_handler(content_types='text')
def message_reply(message):
    global show_bike
    free_bike = ()

    NAME.append(message.text)

    if len(NAME) > 0:
        markup = types.ReplyKeyboardMarkup(resize_keyboard=True)
        item1 = types.KeyboardButton("Show Bikes")
        markup.add(item1)
        bot.send_message(message.chat.id, 'Select an option',
                         reply_markup=markup)

        if message.text == "Show Bikes":
            cursor = db.cursor()
            cursor.execute(
                "SELECT * FROM bike_monitor")
            bike_list = cursor.fetchall()
            db.commit()

            # Printing list of bikes
            for bike in bike_list:
                if bike[4] == 0 or bike[3] == 0:
                    show_bike += ">> Bike #" + \
                        str(bike[0]) + " taken at: " + str(bike[2]) + "\n\n"
                else:
                    show_bike += (">> Bike #" +
                                  str(bike[0]) + " is free") + "\n\n"
                    free_bike = (bike,) + (free_bike)

            bot.send_message(message.chat.id, show_bike,
                             parse_mode='Markdown')

            # Adding buttons
            markup = types.ReplyKeyboardMarkup(resize_keyboard=True)
            for bike in free_bike:
                markup.add(types.KeyboardButton(
                    "bike #" + str(bike[0])))
            bot.send_message(
                message.chat.id, 'Please, select a bike', reply_markup=markup)

            # cleaning bike list
            show_bike = ''

        # Selecting bike
        elif message.text[:4] == "bike":

            cursor = db.cursor()

            # take user_id
            cursor.execute(
                "SELECT card_id FROM users where name = '%s'" % NAME[0])
            user_list = cursor.fetchall()
            user_id = str(user_list[0][0])

            # take bike info about user
            cursor.execute(
                f"SELECT * FROM bike_monitor where user_id = \"{user_id}\"")
            userExsist = cursor.fetchall()
            if not userExsist:
                bot.send_message(
                    message.chat.id, 'Please, enter your name', reply_markup=ReplyKeyboardRemove())

                now = datetime.now()
                current_time = now.strftime("%H:%M")
                cursor.execute(
                    f"UPDATE bike_monitor SET user_id = \"{user_id}\", take = \"{current_time}\", reserved = '0' where reserved = '1' and id = \"{message.text[6:]}\" limit 1")
                print("User ", user_id, " was added")
                bot.send_message(
                    message.chat.id, 'You have reserved bike #'+str(message.text[6:]))
            else:
                bot.send_message(
                    message.chat.id, 'You have already reserved bike')
            db.commit()

    print(NAME)

bot.infinity_polling()


import requests
from twilio.rest import Client
import os
import time

url = 'http://api.vk.com/method/'
token = os.environ['sms_token']
account_sid = os.environ['account_sid']
auth_token = os.environ['auth_token']

def get_json(user_id):
    method = url + 'users.get'
    data = {
        'user_ids': user_id,
        'fields': 'online',
        'access_token' : token,
        'v' : '5.103',
    }

    r = requests.get(method, data)
    return r.json()

def sms_sender(mesage):
    client = Client(account_sid, auth_token)
    message = client.messages.create(
        body= mesage,
        from_='+19367553922',
        to='+79298405593'
        )

    return message.sid

def get_status():
    r = get_json('zyoma')
    status = r['response'][0]['online']
    return status

def main():
    while True:
        status = get_status()
        if status == 1:
            sms_sender(mesage='User online!')
            break

        time.sleep(5)

if __name__ == '__main__':
    main()


from django.conf import settings
from django.conf.urls import include, url  # noqa
from django.contrib import admin
from django.views.generic import TemplateView

import django_js_reverse.views

urlpatterns = [
    url(r'^admin/', admin.site.urls),
    url(r'^jsreverse/$', django_js_reverse.views.urls_js, name='js_reverse'),

    url(r'^$', TemplateView.as_view(template_name='base.html'), name='home'),
    url(r'', include('battles.urls', namespace='battles')),
    url(r'', include('users.urls')),

    url(r'social/', include('social_django.urls', namespace='social')),
    url(r'^api-auth/', include('rest_framework.urls')),
    url(r'api/', include('battles.endpoints_urls', namespace='api_battles')),

]

if settings.DEBUG:
    import debug_toolbar
    urlpatterns = [
        url(r'^__debug__/', include(debug_toolbar.urls)),
    ] + urlpatterns


import logging

from fastapi import FastAPI, UploadFile
from fastapi.middleware.cors import CORSMiddleware

import pandas as pd

from .convert import xlsx_to_jvfdtm

logging.basicConfig(level=logging.DEBUG)

app = FastAPI()
app.add_middleware(
    CORSMiddleware,
    allow_origins=[
        "http://localhost:8080"
    ],
    allow_credentials=True,
    allow_methods=["*"],
    allow_headers=["*"]
)

@app.post("/convert")
async def convert(file: UploadFile):
    contents = await file.read()
    df = pd.read_excel(contents, header=None)
    xlsx_to_jvfdtm(df)

    return {"ok": "ok"}


from datetime import datetime

from ninja import FilterSchema, Schema
from pydantic import Field, validator

class CourseIn(Schema):
    title: str
    description: str
    slug: str
    language: str
    requirements: str
    what_you_will_learn: str
    level: str
    categories: list[int] | None
    instructors: list[int] | None
    is_published: bool = False

class CourseOut(Schema):
    id: int
    title: str
    description: str
    slug: str
    language: str
    requirements: str
    what_you_will_learn: str
    level: str
    categories: list[int]
    instructors: list[int]
    is_published: bool = False
    created: datetime
    modified: datetime

    @validator('created', 'modified', allow_reuse=True)
    def convert_datetime(cls, value: datetime):
        return value.isoformat()

    @staticmethod
    def resolve_categories(obj):
        return [category.id for category in obj.categories.all()]

    @staticmethod
    def resolve_instructors(obj):
        return [instructor.id for instructor in obj.instructors.all()]

class CourseFilter(FilterSchema):
    categories: str | None = Field(q='categories__in')
    language: str | None = Field(q='language__in')
    level: str | None = Field(q='level__in')

    @validator('categories', 'language', 'level', allow_reuse=True)
    def split_testing(cls, value):
        return value.split(',')

class CourseUpdate(Schema):
    title: str | None
    description: str | None
    slug: str | None
    language: str | None
    requirements: str | None
    what_you_will_learn: str | None
    level: str | None
    categories: list[int] | None
    instructors: list[int] | None
    is_published: bool | None

class InvalidCategoriesOrInstructors(Schema):
    detail: str = 'invalid instructors or categories'

class CourseRelationIn(Schema):
    course_id: int

class CourseRelationOut(Schema):
    id: int
    creator_id: int
    course_id: int
    done: bool
    created: datetime
    modified: datetime

    @validator('created', 'modified', allow_reuse=True)
    def convert_datetime(cls, value: datetime):
        return value.isoformat()

class CourseRelationUpdate(Schema):
    done: bool


from http import HTTPStatus

import django.test
import django.urls

__all__ = []

class StaticURLTests(django.test.TestCase):
    def test_homepage_endpoint(self):
        response = django.test.Client().get(
            django.urls.reverse("homepage:home"),
        )
        self.assertEqual(response.status_code, HTTPStatus.OK)

    def test_coffee_endpoint(self):
        response = django.test.Client().get(
            django.urls.reverse("homepage:coffee"),
        )
        self.assertIn("Я чайник", response.content.decode("utf-8"))
        self.assertEqual(response.status_code, HTTPStatus.IM_A_TEAPOT)


import datetime
import os
import platform

class Clock:

    def __init__(self):
        self._stopwatch_counter_num = 10800
        self._running = None
        self._alarm_minute = None
        self._alarm_hour = None
        self._seconds = None
        self._minutes = None
        self._hour = None
        self._alarm_time = None
        self._time = None
        self._date = None

    def get_time(self):
        self._time = datetime.datetime.now().strftime("%H:%M:%S")
        self._hour, self._minutes, self._seconds = self._time.split(':')
        return self._time

    def get_date(self):
        self._date = datetime.datetime.now().strftime("%d/%m/%Y")
        return self._date

    def clock_loop(self):
        while True:
            self.get_time()

    def print_time(self):
        self.get_time()
        self.get_date()
        print(self._time, self._date)

    def alarm(self, alarm_time):
        self._alarm_time = alarm_time
        self._alarm_hour, self._alarm_minute = self._alarm_time.split(':')
        self.alarm_loop()

    def alarm_check(self):
        if self._hour == self._alarm_hour and self._minutes == self._alarm_minute:
            self.beep()
            return True
        else:
            return False

    @staticmethod
    def beep():
        for i in range(3):
            if platform.system() == 'Windows':
                winsound.Beep(5000, 1000)
            elif platform.system() == 'Darwin':
                os.system('say Time is Up')
            elif platform.system() == 'Linux':
                os.system('beep -f 5000')

    def alarm_loop(self):
        while self.alarm_check() is False:
            self.get_time()
            self.alarm_check()

    def stopwatch(self, running):
        self._running = running
        while self._running:
            tt = datetime.datetime.fromtimestamp(self._stopwatch_counter_num)
            self._stopwatch_counter_num += 1
            return tt.strftime("%H:%M:%S")

    def reset_stopwatch(self):
        self._stopwatch_counter_num = 10800


import numpy as np
from collections import defaultdict
from scipy import sparse

def trans_matrice(L,D,vois):
    I,J,V=[],[],[]
    for i in range(len(L)):
        for l in vois[L[i]]:
            j=D[l]
            if j>=0:
                I.append(i)
                J.append(j)
                V.append(1./len(vois[L[i]]))
    I=np.array(I)
    J=np.array(J)
    V=np.array(V)
    return(sparse.csc_matrix((V,(J,I)),shape=(len(L),len(L))))
import sys

n=int(sys.argv[1])
K=int(sys.argv[2])

vois=defaultdict(list)
filename = 'percLeo1000/cl_sel1000/perc_voisins_L_1000_'+str(K)+'.dat'
Tm = np.loadtxt(filename)[:]
print(Tm)
for k in range(len(Tm)):
    Tm[k]=[int(Tm[k][i]) for i in range(2)]
for i in range(len(Tm)):
  vois[Tm[i][0]].append(Tm[i][1])

t=100000
L=np.zeros(t)
Err=np.zeros(t)

for r in range(1,10**4+1):
    p_k=[]
    pos_list=[]
    filename = 'percLeo1000/perc/data_n'+str(n)+'_'+str(K)+'/sites'+str(r)+'.dat'
    Tm = np.loadtxt(filename)[:]
    for a in Tm:
        pos_list.append(a)
    filename = 'percLeo1000/perc/data_n'+str(n)+'_'+str(K)+'/end'+str(r)+'.dat'
    c1 = np.loadtxt(filename)
    i0=pos_list.index(c1)
    def moins_un():
       return(-1)
    dic=defaultdict(moins_un)
    for i in range(n):
       dic[pos_list[i]]=i
    M=trans_matrice(pos_list,dic,vois)
    P=np.zeros(n)
    P[i0]=1
    p_k=[1]
    S=0
    for k in range(t):
        P=M.dot(P)
        p_k.append(np.sum(P))
    p_k=np.array(p_k)
    p_k=p_k[:-1]-p_k[1:]
    L+=p_k
    Err+=p_k**2
    if r%10**2==0:
      np.save('percolation/'+str(n)+'/Probabilities_n'+str(n)+', '+str(K),[L,Err])


# -*- coding: utf-8 -*-
from django.conf import settings
from django.utils.encoding import force_text
from django.utils.timezone import get_current_timezone_name
from cms.cache import _get_cache_version, _set_cache_version, _clean_key, _get_cache_key
from cms.utils import get_cms_setting

def _placeholder_cache_key(placeholder, lang):
    cache_key = '%srender_placeholder:%s.%s' % (get_cms_setting("CACHE_PREFIX"), placeholder.pk, str(lang))
    if settings.USE_TZ:
        tz_name = force_text(get_current_timezone_name(), errors='ignore')
        cache_key += '.%s' % tz_name.encode('ascii', 'ignore').decode('ascii').replace(' ', '_')
    return cache_key

def set_placeholder_cache(placeholder, lang, content):
    """
    Caches the rendering of a placeholder
    """
    from django.core.cache import cache
    cache.set(_placeholder_cache_key(placeholder, lang),
              content,
              get_cms_setting('CACHE_DURATIONS')['content'],
              version=_get_cache_version())
    _set_cache_version(_get_cache_version())

def get_placeholder_cache(placeholder, lang):
    """
    Retrieves the cached content of a placeholder
    """
    from django.core.cache import cache

    return cache.get(_placeholder_cache_key(placeholder, lang),
                     version=_get_cache_version())

def clear_placeholder_cache(placeholder, lang):
    from django.core.cache import cache

    cache.delete(_placeholder_cache_key(placeholder, lang), version=_get_cache_version())

def _placeholder_page_cache_key(page_lookup, lang, site_id, placeholder_name):
    base_key = _get_cache_key('_show_placeholder_for_page', page_lookup, lang, site_id)
    return _clean_key('%s_placeholder:%s' % (base_key, placeholder_name))

def get_placeholder_page_cache(page_lookup, lang, site_id, placeholder_name):
    from django.core.cache import cache

    return cache.get(_placeholder_page_cache_key(page_lookup, lang, site_id, placeholder_name),
                     version=_get_cache_version())

def set_placeholder_page_cache(page_lookup, lang, site_id, placeholder_name, content):
    from django.core.cache import cache

    cache.set(_placeholder_page_cache_key(page_lookup, lang, site_id, placeholder_name),
              content,
              get_cms_setting('CACHE_DURATIONS')['content'], version=_get_cache_version())
    _set_cache_version(_get_cache_version())


#!/usr/bin/env python
# -*- coding: utf-8 -*-
# @Time    : 2020/11/9 0:05
# @Author  : SELF-T-YY
# @Site    :
# @File    : classical_sample_ambiguous_shortestPath.py
# @Software: PyCharm

import json
import networkx as nx
import sys
import numpy

fileWritePath = r'../data_forSystem/ieee_visC/IV_ambiguousBetweennessData.json'
fileOriDataPath = r'../data_forSystem/ieee_visC/IVxy.json'
sampleNameList = ['ISRW', 'TIES', 'SRW', 'RES', 'RJ', 'RNS']
sampleRateList = ['5', '10', '15', '20', '25', '30', '35', '40']

with open(fileOriDataPath) as f:
    oriData = json.load(f)
    tempEdgesList = oriData['edges']
    edgesList = [[_['source'], _['target']] for _ in tempEdgesList]
    tempNodesList = oriData['nodes']
    nodesList = [_['id'] for _ in tempNodesList]

    G = nx.Graph()
    G.add_edges_from(edgesList)
    oriShortestPathData = {}
    # 计算原始数据的平均最短路径
    for node in nodesList:
        length = dict(nx.single_target_shortest_path_length(G, node))
        sum = 0
        for _ in length:
            sum += length[_]
        oriShortestPathData[node] = sum / len(length)

    ambiguousData = {}
    for sampleName in sampleNameList:
        tempAmbiguousData_addName_addRate = {}
        for rate in sampleRateList:
            filePath = r'../data_forSystem/ieee_visC/forceData/IV_forceData_' + sampleName + '_rate_' + rate + '.json'
            print(filePath)
            sampleRate = 'rate-' + rate
            with open(filePath) as f1:
                sampleData = json.load(f1)
                tempEdgesList = sampleData['edges']
                edgesList = [[_['source'], _['target']] for _ in tempEdgesList]

                tempNodesList = sampleData['nodes']
                nodesList = [_['id'] for _ in tempNodesList]

                G = nx.Graph()
                G.add_edges_from(edgesList)
                shortestPath = {}
                for node in nodesList:
                    if not G.has_node(node):
                        continue
                    length = dict(nx.single_target_shortest_path_length(G, node))
                    sum = 0
                    for _ in length:
                        sum += length[_]
                    shortestPath[node] = sum/len(length)

                # 计算歧义
                tempAmbiguousData = {_: oriShortestPathData[_] - shortestPath[_] for _ in
                                     shortestPath
                                     if _ in oriShortestPathData}

                # 归一化
                maxVal = 0
                minVal = 100000000
                for _ in tempAmbiguousData:
                    maxVal = max(maxVal, tempAmbiguousData[_])
                    minVal = min(minVal, tempAmbiguousData[_])

                tempAmbiguousData = {_: (tempAmbiguousData[_] - minVal) / (maxVal - minVal) for _ in tempAmbiguousData}

                tempAmbiguousData_addName_addRate[sampleRate] = tempAmbiguousData
            ambiguousData[sampleName] = tempAmbiguousData_addName_addRate

fw = open(fileWritePath, 'w+')
fw.write(json.dumps(ambiguousData))
fw.close()


#mengimport library yang digunakan
from keras.models import load_model
from PIL import Image
from matplotlib import pyplot as plt
import numpy as np

#Membaca gambar menggunakan library PIL, pembaca juga
#dapat menggunakan library lain, seperti OpenCV
gambar1= Image.open("dataset/mnist/testing/0/img_108.jpg")
gambar2= Image.open("dataset/mnist/testing/1/img_0.jpg")
#mengubah ke ndarray numpy
gambar1 = np.asarray(gambar1)
gambar2 = np.asarray(gambar2)

#memanggil model yang telah ditraining sebelumnya
model = load_model('./modelLeNet5.h5')

#memprediksi gambar dg modelnya
#ingat, intensitas perlu dibagi 255 karena saat training jg dibagi 255
#kemudian format input harus (batch size,tinggi gambar, lebar, depth)
#oleh karena itu, kita gunakan reshape seperti di bawah
pred1 = model.predict_classes((gambar1/255).reshape((1,28,28,1)))
pred2 = model.predict_classes((gambar2/255).reshape((1,28,28,1)))

#mengeplot gambar beserta hasil prediksi
plt.figure('gambar1')
plt.imshow(gambar1,cmap='gray')
plt.title('pred:'+str(pred1[0]), fontsize=22)
print("prediksi gambar1:", pred1[0])

plt.figure('gambar2')
plt.imshow(gambar2,cmap='gray')
plt.title('pred:'+str(pred2[0]), fontsize=22)
print("prediksi gambar2:", pred2[0])

plt.show()

edisi2/10.5.2 Menggunakan Model yang telah ditraining.py


import os
import json
import pytz
import logging
from flask_socketio import SocketIO, emit
from flask import (Flask, render_template, request, jsonify)
from pymongo import MongoClient
from threading import Thread, Event
from datetime import datetime
from convxai.utils import *

logging.basicConfig(format='%(asctime)s - %(message)s', level=logging.INFO)

app = Flask(__name__)
app.config["SECRET_KEY"] = "liaopi6u123sdfakjb23sd"
socketio = SocketIO(app, logger=True, engineio_logger=True,
                    async_mode="threading")
mongo = get_mongo_connection()
thread = Thread()
thread_stop_event = Event()
task_mapping = {}

# ########################################
# # Set up paths to save log files
# ########################################
# system_config = parse_system_config_file()
# logFileName = "log_" + datetime.now().astimezone(pytz.timezone('US/Eastern')
#                                                  ).strftime("%m%d%Y_%H%M%S") + ".txt"
# logFile = open(os.path.join(
#     system_config['system']['logfilePath'], logFileName), "a")

########################################
# Threading and MongoDB
########################################

def init_threading() -> None:
    """
    Initiate the thread to monitor interaction and save data into mongoDB database.
    """
    global thread
    if not thread.is_alive():
        thread = Thread(
            name='mongo-monitoring',
            target=mongo_monitoring,
        )
        thread.start()
        logging.info("Starting Mongo Monitoring Thread")

def mongo_monitoring():
    """
    Set up mongoDB monitoring.
    """
    global task_mapping
    pipeline = [{
        "$match": {
            "$and": [
                {"operationType": "insert"},
                {"fullDocument.role": "agent"},
                {"fullDocument.reply_to": {"$exists": True}}
            ]
        }
    }]
    with mongo.message.watch(pipeline) as stream:
        for insert_change in stream:
            source_message_id = insert_change["fullDocument"]["reply_to"]
            response_text = insert_change["fullDocument"]["text"]
            message_type = insert_change["fullDocument"]["payload"]["message_type"]
            if message_type == "task":
                socketio.emit(
                    "task_response",
                    {"text": response_text},
                    to=task_mapping[source_message_id],
                    namespace="/connection"
                )
            elif message_type == "conv":
                writingIndex = insert_change["fullDocument"]["payload"]["writingIndex"]
                socketio.emit(
                    "conv_response",
                    {"text": response_text, "writingIndex": writingIndex},
                    to=task_mapping[source_message_id],
                    namespace="/connection"
                )
            del task_mapping[source_message_id],

########################################
# Flask Implementation
########################################

def get_data():
    """
    Get the request data.
    """
    data = json.loads(str(request.data, encoding='utf-8'))
    return data

@app.route("/")
def index():
    return render_template("user_interface.html")

@app.route("/init_conversation", methods=["POST"])
def init_conversation():
    """
    Initiate the conversation.
    """
    body = get_data()
    res = mongo.message.insert_one({
        "text": body["text"],
        "role": "user",
        "init": True,
        "time": datetime.now(),
        "conversation_id": body.get("conversation_id"),
    })
    return jsonify({"text": "hello world", "conversation_id": str(res.inserted_id)})

@app.route("/reset")
def reset():
    """
    Reset the interface.
    """
    data = json.dumps({"text": "[RESET]"})
    return jsonify({"text": "RESET"})

########################################
# SocketIO Implementation
########################################

@socketio.on('connect', namespace="/connection")
def test_connect():
    emit('connected', {'data': 'Connected'})
    init_threading()

@socketio.on('disconnect', namespace="/connection")
def test_disconnect():
    logging.info('Client disconnected')

@socketio.on("interact", namespace="/connection")
def interact_socket(body):
    """
    Interaction between the web server and interface via socketIO.
    """
    result = mongo.message.insert_one({
        "text": body["text"],
        "role": "user",
        "init": False,
        "time": datetime.now(),
        "conversation_id": body["conversation_id"],
        "payload": {
            "message_type": body["message_type"],
            "writing_model": body.get("writing_model", None),
        },
        "note": "socket",
        "user_id": body.get("user_id", ""),
    })

    # log text editing results into mongo
    mongo.log.insert_one({
        "user_id": body.get("user_id", ""),
        "time": datetime.now(),
        "text": body["text"],
        "type": body["message_type"],
        "writing_model": body.get("writing_model", None),
        "conversation_id": body["conversation_id"],
    })

    ### Write logs into files ###
    task_mapping[result.inserted_id] = request.sid

    responseTime = datetime.now().astimezone(
        pytz.timezone('US/Eastern')).strftime("%m-%d-%Y %H:%M:%S")
    responseText = body["text"]
    responseMessageType = body["message_type"]
    responseWritingModel = body.get("writing_model", None)

    # logEntry = responseTime + "  Text: " + responseText + "  \n\t\tMessageType:" + \
    #     responseMessageType + "  \n\t\tWritingModel: " + responseWritingModel + "\n"
    # logFile.write(logEntry+"\n")
    # logFile.close()
    # logging.info("Written to file" + logEntry)

@socketio.on("save", namespace="/connection")
def save(body):
    # log text editing results into mongo
    res = mongo.log.insert_one({
        "user_id": body.get("user_id", ""),
        "time": datetime.now(),
        "text": body["text"],
        "event_type": body.get("mode", "auto-save"),
    })

    if res:
        emit('save', {"status": "success", "mode": body.get("mode", "auto-save")})

########################################
# Run Flask and SocketIO.
########################################
def run_flask_socketio():
    """
    Run Flask and Websocket.
    """
    socketio.run(app, host="0.0.0.0", port=8080, debug=False)

    # for debugging
    # socketio.run(app, host="0.0.0.0", port=8001, debug=True)

if __name__ == "__main__":
    init_threading()
    run_flask_socketio()


"""
@file
@brief Data aggregation for timeseries.
"""
import datetime
import pandas
from pandas.tseries.frequencies import to_offset

def _get_column_name(df, name='agg'):
    """
    Returns a unique column name not in the existing dataframe.

    @param      df      dataframe
    @param      name    prefix
    @return             new column name
    """
    while name in df.columns:
        name += '_'
    return name

def aggregate_timeseries(df, index='time', values='y',
                         unit='half-hour', agg='sum',
                         per=None):
    """
    Aggregates timeseries assuming the data is in a dataframe.

    @param      df      dataframe
    @param      index   time column
    @param      values  value or values column
    @param      unit    aggregate over a specific period
    @param      sum     kind of aggregation
    @param      per     second aggregation, per week...
    @return             aggregated values
    """
    if df is None:
        if len(values.shape) == 1:
            df = pandas.DataFrame(dict(time=index, y=values))
            values = 'y'
        else:
            df = pandas.DataFrame(dict(time=index))
            for i in range(values.shape[1]):
                df['y%d' % i] = values[:, i]
            values = list(df.columns)[1:]
        index = 'time'

    def round_(serie, freq, per):
        fr = to_offset(freq)
        res = pandas.DatetimeIndex(serie).floor(fr)  # pylint: disable=E1101
        if per is None:
            return res
        if per == 'week':
            pyres = res.to_pydatetime()
            return pandas.to_timedelta(
                map(
                    lambda t: datetime.timedelta(
                        days=t.weekday(), hours=t.hour, minutes=t.minute),
                    pyres))
        if per == 'month':
            pyres = res.to_pydatetime()
            return pandas.to_timedelta(
                map(
                    lambda t: datetime.timedelta(
                        days=t.day, hours=t.hour, minutes=t.minute),
                    pyres))
        raise ValueError(  # pragma: no cover
            f"Unknown frequency '{per}'.")

    agg_name = _get_column_name(df)
    df = df.copy()
    if unit == 'half-hour':
        freq = datetime.timedelta(minutes=30)
        df[agg_name] = round_(df[index], freq, per)
    else:
        raise ValueError(  # pragma: no cover
            f"Unknown time unit '{unit}'.")
    if not isinstance(values, list):
        values = [values]
    if agg == 'sum':
        gr = df[[agg_name] + values].groupby(agg_name, as_index=False).sum()
        agg_name = _get_column_name(gr, 'week' + index)
        gr.columns = [agg_name] + list(gr.columns[1:])
    elif agg == 'norm':
        gr = df[[agg_name] + values].groupby(agg_name, as_index=False).sum()
        agg_name = _get_column_name(gr, 'week' + index)
        agg_cols = list(gr.columns[1:])
        gr.columns = [agg_name] + agg_cols
        for c in agg_cols:
            su = gr[c].sum()
            if su != 0:
                gr[c] /= su
    else:
        raise ValueError(  # pragma: no cover
            f"Unknown aggregation '{agg}'.")
    return gr.sort_values(agg_name).reset_index(drop=True)


import json
import pytz
import datetime

def lambda_handler(event, context):
    tempvar2=[]
    timezones = pytz.common_timezones
    #timezones = pytz.all_timezones
    for timezone in timezones:
        tempvar = timezone[0:2]
        if tempvar == 'US':
            tempvar2.append(timezone)
    return {
        'statusCode': 200,
        'body': {"TimeZones": tempvar2}
    }


import torch
import torch.nn as nn
import torch.nn.functional as F
# Parts of these codes are from: https://github.com/Linfeng-Tang/SeAFusion

class Sobelxy(nn.Module):
    def __init__(self):
        super(Sobelxy, self).__init__()
        kernelx = [[-1, 0, 1],
                   [-2, 0, 2],
                   [-1, 0, 1]]
        kernely = [[1, 2, 1],
                   [0, 0, 0],
                   [-1, -2, -1]]
        kernelx = torch.FloatTensor(kernelx).unsqueeze(0).unsqueeze(0)
        kernely = torch.FloatTensor(kernely).unsqueeze(0).unsqueeze(0)
        self.weightx = nn.Parameter(data=kernelx, requires_grad=False).cuda()
        self.weighty = nn.Parameter(data=kernely, requires_grad=False).cuda()

    def forward(self, x):
        b, c, w, h = x.shape
        batch_list = []
        for i in range(b):
            tensor_list = []
            for j in range(c):
                sobelx_0 = F.conv2d(torch.unsqueeze(torch.unsqueeze(x[i, j, :, :], 0), 0), self.weightx, padding=1)
                sobely_0 = F.conv2d(torch.unsqueeze(torch.unsqueeze(x[i, j, :, :], 0), 0), self.weighty, padding=1)
                add_0 = torch.abs(sobelx_0) + torch.abs(sobely_0)
                tensor_list.append(add_0)

            batch_list.append(torch.stack(tensor_list, dim=1))

        return torch.cat(batch_list, dim=0)

class Fusionloss(nn.Module):
    def __init__(self):
        super(Fusionloss, self).__init__()
        self.sobelconv = Sobelxy()
        self.mse_criterion = torch.nn.MSELoss()

    def forward(self, image_vis, image_ir, generate_img):
        image_y = image_vis
        B, C, W, H = image_vis.shape
        image_ir = image_ir.expand(B, C, W, H)
        x_in_max = torch.max(image_y, image_ir)
        loss_in = F.l1_loss(generate_img, x_in_max)
        # Gradient
        y_grad = self.sobelconv(image_y)
        ir_grad = self.sobelconv(image_ir)
        B, C, K, W, H = y_grad.shape
        ir_grad = ir_grad.expand(B, C, K, W, H)
        generate_img_grad = self.sobelconv(generate_img)
        x_grad_joint = torch.maximum(y_grad, ir_grad)
        loss_grad = F.l1_loss(generate_img_grad, x_grad_joint)

        return loss_in, loss_grad


# ========================
# Panda3d - panda3d_gpu.py
# ========================

# Panda3d imports.
from panda3d.core import NodePath, ClockObject, Filename, Texture
from panda3d.core import Shader, ShaderAttrib, PNMImage
from panda3d.core import LVector3i

'''# Local imports.
from etc import _path'''

# Basic Timer object for local script invocation.

class TimeIt:

    def __init__(self, msg=""):
        self.__msg = msg
        self.__clock = ClockObject()
    def __enter__(self):
        self.__start_dt = self.__clock.getRealTime()
        return self
    def __exit__(self, *e_info):
        self.__dur = round(self.__clock.getRealTime()-self.__start_dt, 3)
        if self.__msg:
            print()
            print("{}:  {}".format(self.__msg, self.__dur, 3))
            for attr in self.__dict__:
                if attr.startswith("_TimeIt__"): continue
                print("  {}:  {}".format(attr, round(self.__dict__[attr], 3)))
        self.total_time = self.__dur

class GPU_Image:

    lib_path = "/e/dev/solex/gpu/gpu_image_lib.glsl"

    def __init__(self, ref_img,
                       workgroup_size = LVector3i(32,32,1),
                       img_format = Texture.FRgba8,
                       print_times = False):

        self.workgroup_size = workgroup_size
        self.img_format = img_format
        self.x_size = ref_img.getXSize()
        self.y_size = ref_img.getYSize()
        self.z_size = 1
        self.prepare_time = 0
        self.process_time = 0
        self.extract_time = 0

        self.__NP = NodePath("gpu")
        self.__gsg = base.win.getGsg()
        self.__ref_tex = self.__get_Texture(ref_img)
        self.__LINES = self.__Setup()
        self.__print_times = print_times

    def __enter__(self):
        self.process_time = 0
        self.extract_time = 0
        return self

    def __exit__(self, *e_info):
        if self.__print_times:
            total_time = round(self.prepare_time+self.process_time+self.extract_time, 3)
            prep_time = round(self.prepare_time, 3)
            proc_time = round(self.process_time, 3)
            extr_time = round(self.extract_time, 3)
            print()
            print("GPU_Image total time: {}".format(total_time))
            print("  prepare: {}  ({}%)".format(prep_time, round(prep_time/total_time*100),2))
            print("  process: {}  ({}%)".format(proc_time, round(proc_time/total_time*100),2))
            print("  extract: {}  ({}%)".format(extr_time, round(extr_time/total_time*100),2))

    def __get_Texture(self, ref_img):
        # Convert ref_img into texture.
        with TimeIt() as prep_timer:
            ref_tex = Texture()
            # Ensure ref image has an alpha channel.
            if not ref_img.hasAlpha():
                ref_img.addAlpha()
                ref_img.alphaFill(1.0)
            # Load tex and set format
            ref_tex.load(ref_img)
            ref_tex.setFormat(self.img_format)
        self.prepare_time += round(prep_timer.total_time, 3)
        return ref_tex

    def __Setup(self):
        """Prepares GPU_Image obj to receive python calls to
        the shader library."""

        # Open the shader as a file and get lines so we
        # can extract some setup info from it.
        shader_os_path = Filename(self.lib_path).toOsLongName()
        with open(shader_os_path, "r") as shader_file:
            lines = list(shader_file.readlines())

        # Extract lib function names.
        for line in lines:
            # Each function within the image_lib is defined within the confines
            # of an "#ifdef/#endif" block; the name we need immediately follows
            # the "#ifdef" keyword. This name gets mapped directly to "self"
            # as an alias for "self.__Call" so that the user can simply call
            # the shader function as though it were a regular method of "self".
            if line.startswith("#ifdef"):
                func_name = line.split(" ")[1].strip()

                # Setup callback that redirects to self.__Call each time
                # "self.<func_name> is called, passing along arguments;
                # return the modified image (as a Texture object).
                def call(func_name=func_name, **kwargs):
                    mod_tex = self.__Call(str(func_name), **kwargs)
                    return mod_tex

                # Map "func_name" directly to "self".
                self.__dict__[func_name] = call

        # Add workgroup size layout declaration.
        wg = self.workgroup_size
        wg_str = "layout (local_size_x={}, local_size_y={}) in;\n"
        wg_line = wg_str.format(wg.x, wg.y)
        lines.insert(8, wg_line)
        return lines

    def __Call(self, func_name, **kwargs):
        """Receive python call and redirect request to relevant
        function in image shader library; return modified image."""

        # Copy self.__Lines (need to keep orig for further calls) and
        # add "#define" statement to top to trigger compilation of
        # relevant "#ifdef/def" function block in shader.

        lines = list(self.__LINES)
        lines.insert(2, "#define {}".format(func_name))

        # Assemble lines into shader str and compile.
        shader_str = "".join(lines)
        self.__NP.setShader(Shader.makeCompute(Shader.SL_GLSL, shader_str))

        # Set block size from workgroup size.
        block_x = int(self.x_size/self.workgroup_size.x)
        block_y = int(self.y_size/self.workgroup_size.y)
        block_z = int(self.z_size/self.workgroup_size.z)
        block_size = LVector3i(block_x,block_y,block_z)

        # Create mod_tex for GPU.
        with TimeIt() as prep_timer:
            mod_img = PNMImage(self.x_size, self.y_size, 4)
            mod_tex = Texture()
            mod_tex.load(mod_img)
            mod_tex.setMinfilter(Texture.FTLinear)
            mod_tex.setFormat(self.img_format)
        self.prepare_time += prep_timer.total_time

        # Pass textures to shader.
        self.__NP.setShaderInput("ref_tex", self.__ref_tex)
        self.__NP.setShaderInput("mod_tex", mod_tex)

        # Set any additional required inputs for this function.
        for input_name, input_val in list(kwargs.items()):
            if type(input_val) == PNMImage:
                input_val = self.__get_Texture(input_val)
            self.__NP.setShaderInput(input_name, input_val)

        # Call function in shader library.
        shader_attrib = self.__NP.getAttrib(ShaderAttrib)
        with TimeIt() as proc_timer:
            base.graphicsEngine.dispatch_compute(block_size, shader_attrib, self.__gsg)
        self.process_time += proc_timer.total_time

        # Extract modified texture from GPU.
        with TimeIt() as extract_timer:
            base.graphicsEngine.extractTextureData(mod_tex, self.__gsg)
        self.extract_time += extract_timer.total_time
        return mod_tex


import io
import pandas as pd
import requests
from mage_ai.data_preparation.shared.secrets import get_secret_value
if 'data_loader' not in globals():
    from mage_ai.data_preparation.decorators import data_loader
if 'test' not in globals():
    from mage_ai.data_preparation.decorators import test

@data_loader
def load_data_from_api(*args, **kwargs):
    """
    Template for loading data from API
    """
    api_key = get_secret_value('API_Key')
    # Create a dictionary with the API key as a parameter
    headers = {'x-apisports-key': api_key}
    years = ['2021', '2022']
    id = []
    for year in years:
        url = f'https://v3.football.api-sports.io/teams?country=England&league=39&season={year}'
        response = requests.get(url,headers=headers)
        data = response.json()

        data_body = data['response']

        for entry in data_body:
            for teams, content in entry.items():
                if teams == 'team':
                    for item, contents in content.items():
                        if item == 'id':
                            id.append([contents, year])

    df = pd.DataFrame(id, columns=['Team ID', 'Year'])
    return df

@test
def test_output(output, *args) -> None:
    """
    Template code for testing the output of the block.
    """
    assert output is not None, 'The output is undefined'


#!/usr/bin/env python
# coding: utf-8

# In[174]:

import pandas as pd
import seaborn as sns
import matplotlib.pyplot as plt
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import StandardScaler
from sklearn.model_selection import GridSearchCV
from itertools import combinations

# ### Part 1: Get the wine dataset¶
# Describe the dataset
#
# Load in the training and test splits of the dataset

# In[2]:

df_train = pd.read_csv('/Users/eiwi/Practical-ML-DS/Chapter 1. Machine Learning Foundations/DATA/winequality-red-train.csv',index_col=0)
df_test = pd.read_csv('/Users/eiwi/Practical-ML-DS/Chapter 1. Machine Learning Foundations/DATA/winequality-red-test.csv', index_col=0)

# In[3]:

# The charts showing the relationship between each feature and the target
fig, ax = plt.subplots(len(df_train.columns),1, sharey = True, figsize = (18,18))
for i in range(len(df_train.columns)):
    current_col = df_train[df_train.columns[i]]
    ax[i].barh(df_train['quality'], current_col)
    ax[i].set_xlabel(str(df_train.columns[i]))
    ax[i].set_ylabel('quality')
plt.show()

# In[4]:

f, ax = plt.subplots(figsize=(12, 10))
ax = sns.heatmap(df_train.corr(), annot=True)

# In[5]:

print("All the features given in this dataset are numeric.")
print('The most correlated features are: fixed acidity, density and citric acid, total sulfur dioxide and free sulfur dioxide, negatively correlated: fixed acidity and pH, citric and volatile acidity.')

# In[6]:

print('The Quality paremeter is mostly correlated to alcohol (positively) and volatile acidity (negatively).')

# In[7]:

# Separate target and predictors:

# In[8]:

target_train = df_train.pop('quality')
target_test = df_test.pop('quality')

# In[9]:

## Further split train dataset into train and validation sets

X_train, X_validation, y_train, y_validation = train_test_split(df_train, target_train, test_size=0.33, random_state=42)

# In[10]:

# Set the indices in the new datasets as the subquential range of numbers

def reset_indices(df,target):
    lst_idx = []
    for idx,_ in enumerate(df.iloc[:,0]):
        lst_idx.append(idx)
    df['index'] = lst_idx
    df.set_index(df['index'], inplace = True)
    df.drop(columns='index', inplace=True)
    target.index = lst_idx
    return df, target

# In[11]:

X_train, y_train = reset_indices(X_train, y_train)

# In[14]:

X_validation, y_validation = reset_indices(X_validation, y_validation)

# In[15]:

X_test, y_test = reset_indices(df_test, target_test)

# In[16]:

print(len(X_train))
print(len(y_train))
print(len(X_validation))
print(len(y_validation))
print(len(X_test))
print(len(y_test))

# In[17]:

print('The data: train set of {} rows, validation set of {} rows, test set of {} rows.'.format(len(X_train), len(X_validation), len(df_test)), end = '\n')
print('Number of predictors: {}. List of predictors: {}.  Target: {}. Possible classes : {}.'.format(len(X_train.columns), ', '.join(X_train.columns), 'column "quality"', str(set(target_train))), end = '\n')
print('Missing values for train set: {}, for validation set: {}, for test set: {}.'.format(X_train.isna().sum().sum(), X_validation.isna().sum().sum(), df_test.isna().sum().sum()))

# In[18]:

# The data requires scaling due to the difference in units:
X_train.describe()

# In[19]:

# Data scaling
scaler = StandardScaler()
X_train_std = pd.DataFrame(scaler.fit_transform(X_train), columns = X_train.columns)
X_validation_std = pd.DataFrame(scaler.fit_transform(X_validation),  columns = X_validation.columns)
X_test_std = pd.DataFrame(scaler.fit_transform(df_test), columns = df_test.columns)

# In[20]:

fig, ax = plt.subplots(1,2, figsize = (12,8))
ax[0].plot(X_train)
ax[0].set_title('Features before scaling')
ax[1].plot(X_train_std)
ax[1].set_title('Features after scaling')
plt.legend(X_train.columns, loc="lower center", bbox_to_anchor=(0, -0.3),  ncol= 3)
plt.show()

# In[21]:

# Now, the data is ready to be fed to the model.
# I will treat this as a classification problem and will try several basic models to
# get an idea of the most suitable algorithm and hyper parameters.

# ### Part 2: Fit models to the wine dataset and test performance
# Make sure you are comfortable with passing the data through a model.
#
# Evaluate the performance of the model.
#
# Make sure you are testing it on the right set of data.
#

# In[23]:

results_dict = {'model':[],
                'params':[],
                'train_score':[],
                'validation_score':[]
               }

# In[24]:

from sklearn.linear_model import LogisticRegression
from sklearn.svm import SVC
from sklearn.neural_network import MLPClassifier
from sklearn.neighbors import KNeighborsClassifier
from sklearn.tree import DecisionTreeClassifier

models_lst = [
    LogisticRegression(random_state=42),
    SVC(gamma=2, random_state=42),
    MLPClassifier(max_iter=1000, random_state=42),
    KNeighborsClassifier(n_jobs=-1),
    DecisionTreeClassifier(random_state=42),
   ]

# In[25]:

params = [
    {'penalty':('l1','l2', 'elasticnet', 'none'), 'C':(0.025, 0.01,1,10), 'solver': ('newton-cg', 'lbfgs', 'liblinear','sag','saga')},
    {'C':(0.025, 0.01,1,10), 'kernel':('linear', 'rbf'), 'decision_function_shape':('ovr','ovo')},
    {'activation':('relu', 'logistic'), 'solver':('sgd','lbfgs'), 'alpha':(0.001,0.01,0.1,1), 'learning_rate': ('adaptive','constant')},
    {'n_neighbors':(3,5,10), 'weights':('distance','uniform'), 'algorithm':('auto','ball_tree','kd_tree')},
    {'criterion':('gini','entropy'), 'splitter':('best','random'),'max_depth':(5,10,25), 'min_samples_split':(2,3,5)}
]

# In[26]:

for i in range(len(models_lst)):
    model = models_lst[i]
    parameters = params[i]
    grid_search_model = GridSearchCV(model, parameters)
    grid_search_model.fit(X_train,y_train)
    train_score =  grid_search_model.best_score_
    validation_score = grid_search_model.best_estimator_.score(X_validation,y_validation)
    results_dict['model'].append(grid_search_model.best_estimator_)
    results_dict['params'].append(grid_search_model.best_params_)
    results_dict['train_score'].append(train_score)
    results_dict['validation_score'].append(validation_score)

# In[27]:

results_dict

# In[28]:

classifiers_comparison = pd.DataFrame.from_dict(results_dict)

# In[29]:

classifiers_comparison.to_csv('/Users/eiwi/Practical-ML-DS/Chapter 1. Machine Learning Foundations/PROJECT/classifiers_comparison.csv')

# In[30]:

classifiers_comparison

# In[31]:

best_train_score = classifiers_comparison['train_score'].max()
print('best_train_score: ', best_train_score)
print('best_train_score_model: ', classifiers_comparison['model'][classifiers_comparison['train_score']==best_train_score], end = '\n')

print('\n')

second_best_train_score = list(classifiers_comparison['train_score'].sort_values(ascending = False))[1]
print('second_best_train_score: ', second_best_train_score)
print('second_best_train_score_model: ', classifiers_comparison['model'][classifiers_comparison['train_score']==second_best_train_score], end = '\n')

print('\n')

best_validation_score = classifiers_comparison['validation_score'].max()
print('best_validation_score: ', best_validation_score)
print('best_validation_score_model: ', classifiers_comparison['model'][classifiers_comparison['validation_score']==best_validation_score], end = '\n')

print('\n')

second_best_validation_score = list(classifiers_comparison['validation_score'].sort_values(ascending = False))[1]
print('second_best_validation_score: ', second_best_validation_score)
print('second_best_validation_score_model: ', classifiers_comparison['model'][classifiers_comparison['validation_score']==second_best_validation_score], end = '\n')

# In[32]:

print('So far the best models were {} and {}, so I will try to combine them into an ensemble.'.format('DecisionTreeClassifier', 'KNeighborsClassifier' ))

# ### Part 3: Improve your performance by ensembling some of your models
# Combine the results of more than one model
#
# Evaluate the performance of the ensemble

# In[138]:

from sklearn.ensemble import VotingClassifier, StackingClassifier, AdaBoostClassifier, BaggingClassifier, RandomForestClassifier

# In[139]:

ensembles_results = {
    'ensemble':[],
    'train_score':[],
    'validation_score':[]
}

# In[140]:

# First, I will combine DecisionTreeClassifier + KNeighborsClassifier
# using ensemble called VotingClassifier a) with hard vote b) with soft vote

# In[141]:

# a) VotingClassifier with a hard vote: the classifier uses
# predicted class labels for majority rule voting.

# In[142]:

knn_estimator =  KNeighborsClassifier(algorithm='auto', leaf_size=30, metric='minkowski',
                       metric_params=None, n_jobs=-1, n_neighbors=10, p=2)

dtc_estimator = DecisionTreeClassifier(ccp_alpha=0.0, class_weight=None, criterion='entropy',
                         max_depth=25, max_features=None, max_leaf_nodes=None,
                         min_impurity_decrease=0.0, min_impurity_split=None,
                         min_samples_leaf=1, min_samples_split=3,
                         min_weight_fraction_leaf=0.0,splitter='best')

# In[143]:

hard_voting_ensemble = VotingClassifier(estimators=[('knn_estimator',knn_estimator), ('dtc_estimator',dtc_estimator)],
                                   voting='hard')

# In[144]:

hard_voting_ensemble.fit(X_train, y_train)

ensembles_results['ensemble'].append('VotingClassifier_hard')
ensembles_results['train_score'].append(hard_voting_ensemble.score(X_train, y_train))
ensembles_results['validation_score'].append(hard_voting_ensemble.score(X_validation,y_validation))

# In[145]:

# b) VotingClassifier with a soft vote: predicts the class label based on the argmax
# of the sums of the predicted probabilities

# In[146]:

soft_voting_ensemble = VotingClassifier(estimators=[('knn_estimator',knn_estimator), ('dtc_estimator',dtc_estimator)],
                                   voting='soft')

# In[147]:

soft_voting_ensemble.fit(X_train, y_train)

ensembles_results['ensemble'].append('VotingClassifier_soft')
ensembles_results['train_score'].append(soft_voting_ensemble.score(X_train, y_train))
ensembles_results['validation_score'].append(soft_voting_ensemble.score(X_validation,y_validation))

# In[148]:

# Secondly, I will use StackingClassifier with the same DecisionTreeClassifier + KNeighborsClassifier

stacking_ensemble = StackingClassifier(estimators=[('knn_estimator',knn_estimator), ('dtc_estimator',dtc_estimator)])

stacking_ensemble.fit(X_train, y_train)

ensembles_results['ensemble'].append('StackingClassifier')
ensembles_results['train_score'].append(stacking_ensemble.score(X_train, y_train))
ensembles_results['validation_score'].append(stacking_ensemble.score(X_validation,y_validation))

# In[149]:

# Compare StackingClassifier with final AdaBoostClassifier
ada_stacking_ensemble = StackingClassifier(estimators=[('knn_estimator',knn_estimator),
                                                       ('dtc_estimator',dtc_estimator)],
                                           final_estimator=None,
                                           stack_method='auto',
                                           n_jobs=-1)

ada_stacking_ensemble.fit(X_train, y_train)

ensembles_results['ensemble'].append('StackingClassifier_AdaBoost')
ensembles_results['train_score'].append(ada_stacking_ensemble.score(X_train, y_train))
ensembles_results['validation_score'].append(ada_stacking_ensemble.score(X_validation,y_validation))

# In[150]:

# Then compare ensemble perfomance with a BaggingClassifier
# using multiple KNeighborsClassifiers OR DecisionTreeClassifiers

# In[151]:

# a) BaggingClassifier with DecisionTreeClassifier

# In[152]:

dtc_bagging_ensemble = BaggingClassifier(base_estimator = dtc_estimator,
                                         n_estimators=100,
                                         random_state=42)

dtc_bagging_ensemble.fit(X_train, y_train)

# In[153]:

ensembles_results['ensemble'].append('DTC_BaggingClassifier')
ensembles_results['train_score'].append(dtc_bagging_ensemble.score(X_train, y_train))
ensembles_results['validation_score'].append(dtc_bagging_ensemble.score(X_validation,y_validation))

# In[154]:

# b) BaggingClassifier with KNeighborsClassifiers

# In[155]:

knn_bagging_ensemble = BaggingClassifier(base_estimator=knn_estimator,
                                     n_estimators=100,
                                     random_state=42)

# In[156]:

knn_bagging_ensemble.fit(X_train, y_train)

ensembles_results['ensemble'].append('KNN_BaggingClassifier')
ensembles_results['train_score'].append(knn_bagging_ensemble.score(X_train, y_train))
ensembles_results['validation_score'].append(knn_bagging_ensemble.score(X_validation,y_validation))

# In[157]:

# As DecisionTreeclassifier was the most cussessful in previous ensembles,
# now I will try out RandomForestClassifier

# In[158]:

rfc_ensemble = RandomForestClassifier(n_estimators=100, random_state=0)
rfc_ensemble.base_estimator = dtc_estimator

# In[159]:

rfc_ensemble.fit(X_train, y_train)

ensembles_results['ensemble'].append('RandomForestClassifier')
ensembles_results['train_score'].append(rfc_ensemble.score(X_train, y_train))
ensembles_results['validation_score'].append(rfc_ensemble.score(X_validation,y_validation))

# In[160]:

#Compare with AdaBoostClassifier based on DecisionTreeClassifiers only
# (as KNeighborsClassifier doesn't support sample_weight)

# In[161]:

dtc_ada_ensemble = AdaBoostClassifier(base_estimator= dtc_estimator, n_estimators=50,
                                      learning_rate=1.0, algorithm='SAMME.R',
                                      random_state=42)
dtc_ada_ensemble.fit(X_train, y_train)

ensembles_results['ensemble'].append('AdaBoost_dtc_ensemble')
ensembles_results['train_score'].append(dtc_ada_ensemble.score(X_train, y_train))
ensembles_results['validation_score'].append(dtc_ada_ensemble.score(X_validation, y_validation))

# In[162]:

# Now let's compare these ensembles

# In[163]:

ensembles_results

# In[164]:

ensembles_comparison = pd.DataFrame.from_dict(ensembles_results)

# In[165]:

ensembles_comparison

# In[166]:

# Conclusion: the highest validation score of 0.822695 was achieved with DTC_BaggingClassifier
# The second best score of 0.817967 was achieved with RandomForestClassifier
# Surprisingly, VotingClassifier and StackingClassifier combining different base models
# have worse perfomanse than ensembles with 1 estimator (RandomForestClassifier, DTC_BaggingClassifier)

# In[362]:

dtc_ada_ensemble.__class__.__name__

# In[324]:

lst = list(combinations([3,6,7], 2))
for t in lst:
    print(list(t))

(1,2)

# ### Part 4: Write an algorithm to find the best combination of models and hyperparameters.
#
# There are an infinite number of ways that you could combine different models with different hyperparameters, but some will perform better than others.
#
# List the different parameters which you test over, as well as the ranges which you test
#
# Describe the search procedure which your algorithm implements

# In[484]:

from sklearn.model_selection import cross_val_score

# In[533]:

def algo_name(est):
    return est.__class__.__name__
def algo_voting_soft_classifier(estimators_list, param_grid_list, N, X, y, scoring='f1'):
    """
        This function searches for best combination of model and hyperparameters to be used in VotingClasifier(soft)

        estimators_list — list of non-fitted estimators
        param_grid_list — list of param grids for GridSearch, must be same size as estimators_list
        N — number of models to use in ensemble
        X — features
        y — target
        scoring — scoring function name (string)
    """

    # Lists must be of same size
    assert len(estimators_list) == len(param_grid_list)

    # Can't use more estimators than we have
    assert N <= len(estimators_list)

    # We will keep list of best fitted estimators
    best_estimators = []

    # Let's GridSearch em all
    for estimator, param_grid in zip(estimators_list, param_grid_list):
        print("Searching best params for " + algo_name(estimator))
        gs = GridSearchCV(estimator, param_grid, scoring=scoring)
        gs.fit(X,y)
        best_estimators.append(gs.best_estimator_)
        print("Best score = " + str(gs.best_score_))

    # Here we have all possible combinations of N estimators
    all_combinations = list(combinations(best_estimators, N))

    # Keep track of best ensemble yet
    top_score = 0
    top_ensemble = None

    # Go over all combinations
    for t in all_combinations:
        voting_estimators = []
        voting_estimators_names = []

        # Construct list of estimators for VotingClassifier
        for estimator in list(t):
            name = algo_name(estimator)
            voting_estimators_names.append(name)
            voting_estimators.append((name, estimator))

        print(f"\n\nChecking combination of {N} estimators:")
        print("\t\n".join(voting_estimators_names))
        # Cross-validating ensemble of N estimators
        soft_voting_ensemble = VotingClassifier(estimators=voting_estimators, voting='soft')
        cross_vals = cross_val_score(soft_voting_ensemble, X,y, scoring=scoring)

        # Selecting top ensemble
        current_score = max(cross_vals)

        print(f"Score ({scoring}) of this combination is {current_score}")
        if (current_score>top_score):
            top_score = current_score
            top_ensemble = soft_voting_ensemble

    return top_ensemble

# In[534]:

estimators_list = [
    LogisticRegression(random_state=42, n_jobs=-1),
    SVC(gamma=2, random_state=42, probability=True),
#     MLPClassifier(max_iter=1000, random_state=42),
#     KNeighborsClassifier(n_jobs=-1),
    DecisionTreeClassifier(random_state=42)
]

param_grids_list = [
    {'penalty':('l1','l2', 'elasticnet', 'none'), 'C':(0.025, 0.01,1,10), 'solver': ('newton-cg', 'lbfgs', 'liblinear','sag','saga')},
    {'C':(0.025, 0.01,1,10), 'kernel':('linear', 'rbf'), 'decision_function_shape':('ovr','ovo')},
#     {'activation':('relu', 'logistic'), 'solver':('sgd','lbfgs'), 'alpha':(0.001,0.01,0.1,1), 'learning_rate': ('adaptive','constant')},
#     {'n_neighbors':(3,5,10), 'weights':('distance','uniform'), 'algorithm':('auto','ball_tree','kd_tree')},
    {'criterion':('gini','entropy'), 'splitter':('best','random'),'max_depth':(5,10,25,50), 'min_samples_split':(2,3,5,10)}
]

# In[535]:

demo_X = X_train[:100]
demo_y = y_train[:100]

# In[536]:

import warnings
warnings.filterwarnings("ignore")
best_voter = algo_voting_soft_classifier(estimators_list, param_grids_list, 2, demo_X, demo_y, scoring='accuracy')
warnings.filterwarnings("default")

# In[518]:

best_voter

# In[519]:

warnings.filterwarnings("ignore")
best_voter_3 = algo_voting_soft_classifier(estimators_list, param_grids_list, 3, demo_X, demo_y, scoring='accuracy')
warnings.filterwarnings("default")

# In[520]:

best_voter_3

# ### Part 5: Present your results
#
# The final part of the project requires you to attempt to summarise your work. It should be clear how we could replicate the results by implementing exactly the same ensemble (models and hyperparameters).
#
# Please try to communicate and display your results with any graphs or charts.
# If you have any insights into why certain ensembles or models perform better of worse than others, and would like to write a paragraph to explain this, we'd love to read it!
# Please also write a summary paragraph that describes the best permutation that you found.

# In[ ]:

# Based on my observation, the individual models  show lower score than ensembles

# In[676]:

print( 'The average score among individual models is equal to {}, maximum validation score - {} (provided by {}).'.format(round(classifiers_comparison['validation_score'].mean(),3), round(classifiers_comparison['validation_score'],3).max(), str(classifiers_comparison[classifiers_comparison['validation_score'] == classifiers_comparison['validation_score'].max()]['model'])[5:27]))

# In[710]:

print('The average score among ensembles is equal to {}, maximum validation score - {} (provided by {}).'.format(round(ensembles_comparison['validation_score'].mean(),3),
                        round(ensembles_comparison['validation_score'].max(),3),
                        str(ensembles_comparison[ensembles_comparison['validation_score'] == ensembles_comparison['validation_score'].max()]['ensemble'])[5:27]))

# In[ ]:

# The most powerful individual models are DecisionTreeClassifier and KNeighborsClassifier.
# The most powerful ensembles are DTC_BaggingClassifier and RandomForestClassifier (both are based on DecisionTreeClassifier)

# In[ ]:

# Single models VS ensembles

# In[718]:

fig, ax = plt.subplots(1, 2, figsize = (16,6), sharey = True)

ax[0].bar(classifiers_comparison.index, classifiers_comparison['train_score'], width = -0.3, align = 'edge', color='navy')
ax[0].bar(classifiers_comparison.index, classifiers_comparison['validation_score'], width = 0.3, align = 'edge', color='orange')
ax[0].set_title('Score of the individual models')

ax[1].bar(ensembles_comparison.index, ensembles_comparison['train_score'], width = -0.3, align = 'edge', color='navy')
ax[1].bar(ensembles_comparison.index, ensembles_comparison['validation_score'], width = 0.3, align = 'edge' , color='orange' )
ax[1].set_title('Score of the ensembles')

plt.title(label = 'Difference in score between individual models and ensembles', loc='left')
plt.show()

# In[719]:

# Interestingly, while individual models tend to show slightly higher score on the validation set,
# while ensembles tend to do the opposite: their train score is higher than validation one.

# In[720]:

# It is also interesting to compare the ensembles combining the different models VS homogeneous ones

# In[727]:

ensembles_comparison

# In[725]:

fig, ax = plt.subplots()
ax.set_title('Ensembles score')
ax.bar(ensembles_comparison.index, ensembles_comparison['train_score'], width = -0.3, align = 'edge', color='navy')
ax.bar(ensembles_comparison.index, ensembles_comparison['validation_score'], width = 0.3, align = 'edge' , color='orange')
ax.set_ylabel('Score')
ax.set_xlabel('Ensemble')

# In[ ]:

# After performing a series of test, I made a conclusion that the best possible score can be achieved with
# the ensemble methods that combine multiple versions of the same estimator (BaggingClassifier, RandomForestClassifier, ) rather than
# the ones combining a list of differents estimators (VotingClassifier and StackingClassifier).

# In[ ]:

# So for the further iteartions I would use BaggingClassifier or RandomForestClassifier.
# The key for reproducibility the would be to stick to the repeatable method for every step of data analysis:
# so it is reasonable to create the standard functions for data processing and cleaning, scaling and modelling.
# Also, in order to replicate the results I would pass an argument 'random_state = 42' to the models
# which allows to get the same results while running the model again.

# ### Part 6:
# A stakeholder asks you which features most affect the response variable (output). Describe how you would organise a test to determine this.

# In[521]:

# I would consider the influence of each feature to the output as a change that occurs in target value
# with change of the output with one unit

# In[525]:

get_ipython().system("['Approach for features importance calculation']('Change_in_var.png')")

# In[745]:

# I would test this approach on desicion-tree based model in order to compare the results with the built function:
feature_importances = list(dtc_estimator.fit(demo_X, demo_y).feature_importances_)
features = demo_X.columns

# In[757]:

feat_imp = pd.DataFrame()
feat_imp['features'] = features
feat_imp['feature_importances'] = feature_importances

# In[762]:

feat_imp.sort_values(by = 'feature_importances', ascending = False)

# In[ ]:

# The inbuilt function suggests that alcohol has the strongest effect on the target (quality of  wine).


from rest_framework import viewsets, mixins
from rest_framework.response import Response
from .models import AssignmentGroup, Assignment, StudentAssignment
from .serializers import (AssignmentGroupSerializer,
                          AssignmentSerializer, StudentAssignmentSerializer)
from .policies import (AssignmentGroupAccessPolicy,
                       AssignmentAccessPolicy, StudentAssignmentAccessPolicy)

class AssignmentGroupViewSet(viewsets.ModelViewSet):
    serializer_class = AssignmentGroupSerializer
    permission_classes = [AssignmentGroupAccessPolicy]

    def retrieve(self, request, *args, **kwargs):
        instance = self.get_object()
        assignments = instance.assignments.all()

        return Response({
            'assignment_group': self.get_serializer(instance).data,
            'assignments': AssignmentSerializer(assignments, many=True).data
        })

    def get_queryset(self):
        queryset = AssignmentGroup.objects.all()

        course = self.request.query_params.get('course')
        if course:
            queryset = queryset.filter(course=course)

        return queryset

class AssignmentViewSet(viewsets.ModelViewSet):
    serializer_class = AssignmentSerializer
    permission_classes = [AssignmentAccessPolicy]

    def retrieve(self, request, *args, **kwargs):
        instance = self.get_object()
        student_assignments = instance.student_assignments.all()

        return Response({
            'assignment': self.get_serializer(instance).data,
            'student_assignments': StudentAssignmentSerializer(
                student_assignments, many=True).data
        })

    def get_queryset(self):
        queryset = Assignment.objects.all()

        course = self.request.query_params.get('course')
        if course:
            queryset = queryset.filter(course=course)

        group = self.request.query_params.get('group')
        if group:
            queryset = queryset.filter(group=group)

        return queryset

class StudentAssignmentViewSet(mixins.RetrieveModelMixin,
                               mixins.UpdateModelMixin,
                               viewsets.GenericViewSet):
    serializer_class = StudentAssignmentSerializer
    permission_classes = [StudentAssignmentAccessPolicy]

    def retrieve(self, request, *args, **kwargs):
        instance = self.get_object()
        assignment = instance.assignment

        return Response({
            'student_assignment': self.get_serializer(instance).data,
            'assignment': AssignmentSerializer(assignment).data
        })

    def get_object(self):
        student = self.request.query_params.get('student')
        assignment = self.request.query_params.get('assignment')

        return StudentAssignment.objects.get(
            student=student, assignment=assignment)


from django.db import models
import auto_prefetch
from django_resized import ResizedImageField
from folio.utils.media import MediaHelper
from folio.utils.choices import PortfolioChoices
from folio.utils.models import  NameBaseModel, ExperienceAndSchoolModel
from django.utils.text import slugify
from django.urls import reverse
from ckeditor.fields import RichTextField
from taggit.managers import TaggableManager
from django.utils.html import mark_safe
from user.models import UserModel

# Create your models here.

class Experiences(ExperienceAndSchoolModel):
    name_of_place = models.CharField(max_length=200 )
    work_as = models.CharField(max_length=200 )

    class Meta:
        verbose_name = 'Exprience'
        verbose_name_plural = 'Experiences'

class Schools(ExperienceAndSchoolModel):
    name_of_school = models.CharField(max_length=200 )
    course = models.CharField(max_length=200 )
    place = models.CharField(max_length=200 )

    class Meta:
        verbose_name = 'School'
        verbose_name_plural = 'Schools'

class AboutMeModel(NameBaseModel):

    email = models.EmailField(null=True)
    phone = models.BigIntegerField(null=True, blank=True)
    location = models.CharField(max_length=200, null=True)
    thumbnail = ResizedImageField(upload_to=MediaHelper.get_image_upload_path, verbose_name="Image")
    resume_photo = ResizedImageField(upload_to=MediaHelper.get_image_upload_path, verbose_name="Image", null=True, blank=True)

class ServicesModel(NameBaseModel):
    i_class = models.CharField(max_length=200)
    description2 = models.TextField(null=True)
    image = ResizedImageField(upload_to=MediaHelper.get_image_upload_path, verbose_name="Image", null=True, blank=True)

    def save(self, *args, **kwargs):
        if not self.slug and self.name:
            self.slug = slugify(self.name)
        return super().save(*args, **kwargs)

    def get_absolute_url(self):
        return reverse('servicedetail',kwargs={'slug':self.slug})

    class Meta:
        verbose_name = 'Service'
        verbose_name_plural = 'Services'

class ServiceRatingModel(NameBaseModel):
    pesentage = models.IntegerField()

class NewsModel(NameBaseModel):
    thumbnail = ResizedImageField(upload_to=MediaHelper.get_image_upload_path, verbose_name="Image")

    def save(self, *args, **kwargs):
        if not self.slug and self.name:
            self.slug = slugify(self.name)
        return super().save(*args, **kwargs)

class Tags(models.Model):
    name = models.CharField( max_length=50)
    def __str__(self):
        return self.name

    class Meta:
        verbose_name = 'Tag'
        verbose_name_plural = 'Tags'

class Images(models.Model):
    files = models.FileField(upload_to=MediaHelper.get_image_upload_path)
    potfolio = auto_prefetch.ForeignKey( "PotfolioModel", on_delete=models.CASCADE, null=True, blank=True)
    news = auto_prefetch.ForeignKey( NewsModel, on_delete=models.CASCADE,  null=True, blank=True)
    blogs = auto_prefetch.ForeignKey( 'PortfolioBlogModel', on_delete=models.CASCADE,  null=True, blank=True)

    def image_tag(self):
        return mark_safe(''%(self.files))
        # return u'' % escape(MediaHelper.get_image_upload_path)
    image_tag.short_description = 'Image'

    class Meta:
        verbose_name = 'Image'
        verbose_name_plural = 'Images'

class PotfolioModel(NameBaseModel):
    client = models.CharField(max_length=200)
    firm_name = models.CharField(max_length=200)
    image = ResizedImageField(upload_to=MediaHelper.get_image_upload_path, verbose_name="Image", null=True, blank=True)
    category = TaggableManager()
    potfoliochoices = models.CharField(choices=PortfolioChoices.choices, max_length=20, default=PortfolioChoices.Backend)
    live_website = models.CharField(max_length=200, null=True, blank=True)
    date_created = models.DateTimeField(null=True, blank=True)

    def get_potfolio_url(self):
        return reverse('servicedetail',kwargs={'slug':self.slug})

    def save(self, *args, **kwargs):
        if not self.slug and self.name:
            self.slug = slugify(self.name)
        return super().save(*args, **kwargs)

    class Meta:
        verbose_name = 'Portfolio'
        verbose_name_plural = 'Portfolios'

class PotfolioQuestionModel(models.Model):
    questions = models.CharField(max_length=200, null=True)
    answer = RichTextField(null=True)

class CategoryModel(models.Model):
    name = models.CharField(max_length=200)
    slug = models.SlugField(null=True,blank=True)

    def __str__(self):
        return self.name

class PortfolioBlogModel(NameBaseModel):
    category = models.ForeignKey(CategoryModel, on_delete=models.CASCADE)
    image = ResizedImageField(upload_to=MediaHelper.get_image_upload_path, verbose_name="Image", null=True, blank=True)
    tags = TaggableManager()

    def save(self, *args, **kwargs):
        if not self.slug and self.name:
            self.slug = slugify(self.name)
        return super().save(*args, **kwargs)

class CommentsModel(models.Model):
    author = models.ForeignKey(UserModel, on_delete=models.CASCADE)
    post = models.ForeignKey(PortfolioBlogModel,  on_delete=models.CASCADE)
    body = models.TextField(help_text='Add a comment')
    created = models.DateTimeField(auto_now_add=True)
    updated = models.DateTimeField(auto_now=True)
    active = models.BooleanField(default=True)

    def __str__(self):
        return f'Comment by {self.author} on {self.post}'


import asyncio
import websockets
import soundfile as sf
import numpy as np

# Initialize a counter for file naming
file_counter = 0
connected_clients = set()

async def register_client(websocket):
    connected_clients.add(websocket)

async def unregister_client(websocket):
    connected_clients.remove(websocket)

async def send_to_all_clients(message, sender):
    for client in connected_clients:
        if client != sender:
            await client.send(message)

async def audio_handler(websocket, path):
    global file_counter  # Use the global counter
    await register_client(websocket)

    try:
        while True:
            audio_data = await websocket.recv()
            # Process audio_data as needed, e.g., save it to a file, perform further analysis, etc.
            print("Received audio data:", len(audio_data), "bytes")

            # Convert the audio data to 'float32'
            audio_data = np.frombuffer(audio_data, dtype='float32')

            # Generate a new file name with a consecutive number
            file_name = f"received_audio_{file_counter}.wav"

            # Save the audio data to the new file
            with sf.SoundFile(file_name, mode="w", samplerate=16000, channels=1, subtype="FLOAT") as f:
                f.write(audio_data)

            file_counter += 1  # Increment the counter

            # Broadcast the received audio to all other clients
            await send_to_all_clients(audio_data.tobytes(), websocket)

    except websockets.ConnectionClosedOK:
        print(f"WebSocket connection closed by the client: {websocket.remote_address}")
    except websockets.ConnectionClosedError:
        print(f"WebSocket connection closed unexpectedly: {websocket.remote_address}")
    finally:
        await unregister_client(websocket)

start_server = websockets.serve(audio_handler, "localhost", 8765)  # WebSocket server listens on localhost:8765

async def main():
    await start_server
    await asyncio.Future()  # Keep the server running indefinitely

asyncio.get_event_loop().run_until_complete(main())


import os
import sys
sys.path.append("Mask_RCNN")

from mrcnn.model import MaskRCNN
from mrcnn import utils
from data import Data
from waldo_config import Waldoconfig

if __name__ == '__main__':
    config = Waldoconfig()
    config.display()

    model = MaskRCNN(mode="training", config=config,
                              model_dir=config.MODEL_DIR)

    weights_path = config.COCO_WEIGHTS_PATH
    # Download weights file
    if not os.path.exists(weights_path):
        utils.download_trained_weights(weights_path)

    # don't load last layers because we are going to train it
    model.load_weights(weights_path, by_name=True, exclude=[
        "mrcnn_class_logits", "mrcnn_bbox_fc",
        "mrcnn_bbox", "mrcnn_mask"])

    dataset_train = Data()
    dataset_train.load(config.DATA_DIR, "train")
    dataset_train.prepare()

    dataset_val = Data()
    dataset_val.load(config.DATA_DIR, "val")
    dataset_val.prepare()

    # training just heads layer is enough
    model.train(dataset_train, dataset_val,
                learning_rate=config.LEARNING_RATE,
                epochs=30,
                layers='heads')