import open3d
from open3d import pipelines
import numpy as np
import os
import argparse
import yaml
import sys

parser = argparse.ArgumentParser(description='Run 3D reconstruction on a sequence of RGB-D images.')
parser.add_argument('-c', '--camera', type=str, required=True, help='path to the camera intrinsics yaml file')
parser.add_argument('-p', '--poses', type=str, required=True, help='path to the camera poses yaml file')
parser.add_argument('-r', '--rgb_folder', type=str, required=True, help='path to the folder containing the RGB images')
parser.add_argument('-d', '--depth_folder', type=str, required=True, help='path to the folder containing the depth images')
args = parser.parse_args()

camera_file = args.camera
poses_file = args.poses
rgb_folder = args.rgb_folder
depth_folder = args.depth_folder

if not os.path.exists(camera_file):
    sys.exit(f"The camera file {camera_file} does not exist.")

if not os.path.exists(poses_file):
    sys.exit(f"The poses file {poses_file} does not exist.")

if not os.path.exists(rgb_folder):
    sys.exit(f"The RGB folder {rgb_folder} does not exist.")

if not os.path.exists(depth_folder):
    sys.exit(f"The depth folder {depth_folder} does not exist.")

with open(camera_file, 'r') as f:
    camera_data = yaml.load(f, Loader=yaml.FullLoader)

# Extract the camera intrinsic parameters
fx = camera_data['camera_matrix']['data'][0]
fy = camera_data['camera_matrix']['data'][4]
cx = camera_data['camera_matrix']['data'][2]
cy = camera_data['camera_matrix']['data'][5]
image_width = camera_data['image_width']
image_height = camera_data['image_height']

# Create the PinholeCameraIntrinsic object
intrinsic = open3d.camera.PinholeCameraIntrinsic(image_width, image_height, fx, fy, cx, cy)
intrinsic.intrinsic_matrix = [[fx, 0, cx], [0, fy, cy], [0, 0, 1]]
print(intrinsic.intrinsic_matrix)
cam = open3d.camera.PinholeCameraParameters()
cam.intrinsic = intrinsic

with open(poses_file, 'r') as f:
    poses_data = yaml.load(f, Loader=yaml.FullLoader)

volume = pipelines.integration.ScalableTSDFVolume(voxel_length=0.5 / 512.0, sdf_trunc=0.005,
                                                 color_type=pipelines.integration.TSDFVolumeColorType.RGB8)

for pose_index, pose_info in poses_data.items():
    quaternion = pose_info['camera_to_world']['quaternion']
    translation = pose_info['camera_to_world']['translation']
    M = np.eye(4)

    # Note: You should normalize the quaternion before converting it to a rotation matrix
    quaternion = np.array([quaternion['w'], quaternion['x'], quaternion['y'], quaternion['z']])
    quaternion /= np.linalg.norm(quaternion)

    M[:3, :3] = open3d.geometry.get_rotation_matrix_from_quaternion(quaternion)
    M[:3, 3] = [translation['x'], translation['y'], translation['z']]

    color_path = os.path.join(rgb_folder, f'{pose_index:06d}.png')
    depth_path = os.path.join(depth_folder, f'{pose_index:06d}.png')

    if not os.path.exists(color_path) or not os.path.exists(depth_path):
        print(f"Missing data for pose {pose_index}, skipping.")
        continue

    print(M)
    color = open3d.io.read_image(color_path)
    depth = open3d.io.read_image(depth_path)
    rgbd = open3d.geometry.RGBDImage.create_from_color_and_depth(color, depth, depth_trunc=1.0, convert_rgb_to_intensity=False)

    volume.integrate(rgbd, cam.intrinsic, np.linalg.inv(M))
    print(f"Processed pose {pose_index}")

mesh = volume.extract_triangle_mesh()
mesh.compute_vertex_normals()

pcd = open3d.geometry.PointCloud()
pcd.points = mesh.vertices

output_folder = os.path.dirname(camera_file)
open3d.io.write_triangle_mesh(os.path.join(output_folder, 'fusion_mesh.ply'), mesh, write_ascii=False,
                              compressed=False, write_vertex_normals=True, write_vertex_colors=True,
                              write_triangle_uvs=True, print_progress=True)
open3d.io.write_point_cloud(os.path.join(output_folder, 'fusion_pointcloud.pcd'), pcd, write_ascii=False,
                            compressed=False, print_progress=False)

open3d.visualization.draw_geometries([mesh])


import datetime
import random
import string

from daddypunmaster3000.models import GameSession

def create_session():

    while True:
        random_id = ''.join(
            random.choice(string.ascii_lowercase + string.digits)
            for _ in range(4)
        )
        try:
            GameSession.objects.get(session_id=random_id)
        except GameSession.DoesNotExist:
            game_status = GameSession(session_id=random_id)
            game_status.save()

            return random_id

def delete_old_sessions():

    sessions = GameSession.objects.all()
    for session in sessions:
        session_delta_date = session.updated + datetime.timedelta(days=1)
        session_naive = session_delta_date.replace(tzinfo=None)
        if session_naive < datetime.datetime.now().replace(tzinfo=None):
            session.delete()


from django.shortcuts import render,HttpResponse
from django.views.generic.list import ListView
import sys
sys.path.append("..")
from ZpModel.models import Zp,CV,CV2Zp,Com
from django.db.models import Q
import re
from functools import reduce
import operator
from django.contrib.auth.models import User

def page_not_found(request):
    return render(request,'404.html')

def loginfo(request):
    user = request.user
    if user != None and user != "" and not user.is_anonymous:
        context = '&nbsp' + str(user) + '' \
                '&nbsp&nbsp&nbsp 注销'
    else:
        context = '&nbsp&nbsp登录'
    return context

def me(request):
    context = {}
    context['user'] = loginfo(request)

    if request.user.is_authenticated:
        u = User.objects.get(username=request.user)
        try:
            cv = CV.objects.get(user_id=u.id)
            context['name'] = u.first_name
            context['sex'] = cv.sex
            context['age'] = cv.age
            context['edu'] = cv.edu
            context['tc'] = cv.super
            context['jn'] = cv.tech
            context['tel'] = cv.tel
            context['email'] = u.email
            context['sal1'] = cv.K1
            context['sal2'] = cv.K2
            context['workingExp']=cv.exp
            context['wantJob']=cv.wantJob
            context['school']=cv.school
            context['gzjl']=cv.gzjl
            context['jybj'] = cv.jybj
            context['zwjs']=cv.zwjs
            for k in context.keys():
                if context[k]==None:
                    context[k]=""
        except:
            if u.last_name == 'pt':
                cv = CV(user_id=u.id)
                cv.save()
            else:
                return ZpListView.as_view()(request)
    else:#如果没有登录就返回注册
        return register(request)
    return render(request,'me.html',context)

def submit_me(request):
    context = {}
    if not request.user.is_authenticated:
        return register(request)

    user = request.GET.get('user')
    sex = request.GET.get('sex')
    edu = request.GET.get('edu')
    age = request.GET.get('age')
    super = request.GET.get('tc')
    tech = request.GET.get('jn')
    tel = request.GET.get('tel')
    email = request.GET.get('email')
    K1 = request.GET.get('sal1')
    K2 = request.GET.get('sal2')
    exp = request.GET.get('workingExp')
    wantJob = request.GET.get('wantJob')
    school= request.GET.get('school')
    gzjl = request.GET.get('gzjl')
    jybj = request.GET.get('jybj')
    zwjs = request.GET.get('zwjs')

    u = User.objects.get(username=request.user)
    cv = CV.objects.get(user_id=u.id)
    cv.age = age if age or None else None

    if exp or None:#None 和 ""都不执行
        cv.exp = exp
    else:
        cv.exp=None
    if K1 or None:
        cv.K1=K1
    else:
        cv.K1=None
    if K2 or None:
        cv.K2 = K2
    else:
        cv.K2=None
    cv.sex = sex
    cv.edu=edu
    cv.super=super
    cv.tech=tech
    cv.tel=tel
    u.email=email
    cv.wantJob=wantJob
    cv.school=school
    cv.gzjl=gzjl
    cv.jybj=jybj
    cv.zwjs=zwjs
    cv.save()

    u.first_name=user
    u.save()
    return me(request)

def register(request):
    context = {}
    context['user'] = loginfo(request)
    return render(request,'register.html',context)

class ZpListView(ListView):
    we=""
    edu=""
    job=""
    salary=""
    path=""
    search=""
    user = "登录"
    model = Zp#数据源
    template_name = "index.html"#网页链接
    paginate_by = 10  # if pagination is desired，分页每页的信息条数

#修改查询数据
    def get_queryset(self):
        d={"edu1":"大专","edu2":"本科","edu3":"硕士","edu4":"博士","edu5":"不限",
           "job1":"教育","job2":"互联网","job3":"管理","job4":"文秘","job5":"电子","job6":"机械","job7":"客服","job8":"销售",
           "job9":"银行","job10":"保险","job11":"汽车","job12":"采购","job13":"化工","job14":"设计",
           "we1":[1,3],"we2":[3,5],"we3":[5,10],"we4":[10,100],
           "sa1":[0,3],"sa2":[3,5],"sa3":[5,10],"sa4":[10,30],"sa5":[30,1000],"sa6":[0,0]}
        r=Zp.objects.all()
        #搜索框
        try:
            self.search=re.match('/list_(.*)',self.path).group(1)
            r = r.filter(jobName__contains=self.search)
        except:
            pass
        #教育等级
        if self.edu!=None and self.edu!="":
            edus=self.edu.split('_')
            eduLevel = [d[x] for x in edus if x in d.keys()]
            if len(eduLevel)==1:
                r = r.filter(eduLevel=eduLevel[0])
            elif len(eduLevel)==0:
                pass
            else:
                r=r.filter(eduLevel__in=eduLevel)
        #工作领域
        if self.job!=None and self.job!="":
            jobs=self.job.split('_')
            jobType = [d[x] for x in jobs if x in d.keys()]
            try:
                r=r.filter(reduce(operator.or_, map(lambda x: Q(jobType__contains=x), jobType)))
            except:
                pass
        #薪资
        if self.salary!=None and self.salary!="":
            k=[]
            for x in self.salary.split('_'):
                if x in d.keys():
                    k.append(d[x])
            r = r.filter(reduce(operator.or_, map(lambda x: Q(K1__gte=x[0]) & Q(K2__lte=x[1]), k)))
        #经验
        if self.we!=None and self.we!="":
            k=[]
            for x in self.we.split('_'):
                if x in d.keys():
                    k.append(d[x])
            r = r.filter(reduce(operator.or_, map(lambda x: Q(W1__gte=x[0]) & Q(W2__lte=x[1]), k)))
        return r

#添加返回数据
    def get_context_data(self, **kwargs):
        context = super().get_context_data(**kwargs)
        if self.user!=None and self.user!="" and not self.user.is_anonymous:
            context['user']='&nbsp'+str(self.user)+'' \
                '&nbsp&nbsp&nbsp 注销'
        else:
            context['user'] = '&nbsp&nbsp登录'
        return context

#获取request
    def get(self, request, *args, **kwargs):
        self.we = request.GET.get('we')
        self.edu = request.GET.get('edu')
        self.job = request.GET.get('job')
        self.salary = request.GET.get('salary')
        self.path = request.path
        self.user = request.user
        r=super().get(self, request, *args, **kwargs)
        return r

def CVgo(request):
    user=request.user
    if not user.is_authenticated:
        return HttpResponse(status=403,content='请登录')
    zpId=request.GET.get('zpId')
    com_id=Zp.objects.get(id=zpId).com_id
    jobName = request.GET.get('jobName')
    cv2zp=CV2Zp(zp_id=zpId,user_id=user.id)
    cv2zp.jobName=jobName
    cv2zp.email=user.email
    cv2zp.name=user.first_name
    cv2zp.com_id=com_id
    try:
        CV.objects.get(user_id=user.id)
    except:
        return HttpResponse("请完善简历")
    try:
        cv2zp.save()
    except:
        return HttpResponse(content="不能重复投递")
    return HttpResponse('OK')


# Selenium Web Scraping
from selenium.webdriver.common.by import By
from selenium.webdriver.common.keys import Keys

import time
import logging

def load_more_items(driver):
    status = True
    while status:
        driver.find_element(By.XPATH, '//body').send_keys(Keys.END)
        time.sleep(4)
        try:
            driver.find_elements(By.ID, "masonryViewMore-link")[0].click()
            logging.info("Found more items")
            time.sleep(4)
        except:
            logging.info("No more items found")
            status = False

def get_links_to_items(driver):
    # Fully load all the items
    load_more_items(driver)
    item_links = []
    # Extract the links to the items
    items = driver.find_elements(By.XPATH, "//div[@class='searchVenue card feed-item']")
    for item in items:
        link = item.find_elements(By.TAG_NAME, "a")[0].get_attribute("href")
        item_links.append(link)
    return item_links

def dismiss_ads(driver):
    try:
        logging.info("Trying to dismiss ads...")
        iframe1 = driver.find_elements(By.XPATH, "//div[@id='aswift_1_host']/iframe")[0]
        driver.switch_to.frame(iframe1)
        iframe2 = driver.find_elements(By.XPATH, "//iframe[@id='ad_iframe']")[0]
        driver.switch_to.frame(iframe2)
        driver.find_elements(By.XPATH, "//div[@id='dismiss-button']")[0].click()
    except:
        pass
        logging.info("No ads found...")

class BurpleExtract:

    def __init__(self, driver):
        self.driver = driver

        self.data_dict = dict()
        self.item_links = []

        self.data_dict["title"] = self._get_title()
        self.data_dict["num_reviews"] = self._get_num_reviews()
        self.data_dict["num_wishlisted"] = self._get_num_wishlisted()
        self.data_dict["area"] = self._get_area()
        self.data_dict["price"] = self._get_price()
        self.data_dict["tags"] = self._get_tags()
        self.data_dict["bio"] = self._get_bio()

    def _get_title(self):
        title = self.driver.find_elements(By.CLASS_NAME, "venue-title")[0].text
        return title

    def _get_num_reviews(self):
        num_reviews = self.driver.find_elements(By.CLASS_NAME, "venue-count-reviews")[0].text
        return num_reviews

    def _get_num_wishlisted(self):
        num_wishlisted = self.driver.find_elements(By.CLASS_NAME, "venue-count-wishlisted")[0].text
        return num_wishlisted

    def _get_area(self):
        area = self.driver.find_elements(By.CLASS_NAME, "venue-area")[0].text
        return area

    def _get_price(self):
        price = self.driver.find_elements(By.CLASS_NAME, "venue-price")[0].text
        return price

    def _get_tags(self):
        tags = self.driver.find_elements(By.CLASS_NAME, "venue-tags")[0].text
        return tags

    def _get_bio(self):
        bio = self.driver.find_elements(By.CLASS_NAME, "venue-bio")[0].text
        return bio

    def _load_more_reviews(self):
        status = True
        while status:
            self.driver.execute_script("window.scrollTo(0, document.body.scrollHeight);")
            time.sleep(4)
            try:
                self.driver.find_elements(By.ID, "load-more-reviews")[0].click()
                logging.info("Found more reviews")
                time.sleep(4)
            except:
                logging.info("No more reviews found")
                status = False

    def get_reviews(self):
        self.review_title = []
        self.review_desc = []
        self.review_image = []

        BurpleExtract._load_more_reviews(self)
        reviews_card = self.driver.find_elements(By.XPATH, "//div[@id='foodMasonry']/div[@class='food card feed-item']")
        logging.info(f"Number of reviews: {len(reviews_card)}")

        if len(reviews_card) == 0:
            self.review_title.append('NA')
            self.review_desc.append('NA')
            self.review_image.append('NA')

        for index, card in enumerate(reviews_card):
            logging.info(f"Review Item: {index}")
            card_content = card.find_elements(By.XPATH, "//div[@class='card-body']")[index]
            image_content = card.find_elements(By.XPATH, "//div[@class='food-image']")[index]
            try:
                review_title = card_content.find_elements(By.CLASS_NAME, "food-content")[0].find_elements(By.CLASS_NAME, "food-description-title")[0].text
                self.review_title.append(review_title)
                logging.info("Review title extracted")
            except:
                logging.warning("No review title found")
                self.review_title.append('NA')

            try:
                review_desc = card_content.find_elements(By.CLASS_NAME, "food-content")[0].find_elements(By.CLASS_NAME, "food-description-body")[0].text
                self.review_desc.append(review_desc)
                logging.info("Review description extracted")
            except:
                logging.warning("No review description found, appending NA to it")
                self.review_desc.append('NA')

            try:
                review_image = image_content.find_elements(By.TAG_NAME, "img")[0].get_attribute("src")
                self.review_image.append(review_image)
                logging.info("Review image extracted")
            except:
                logging.warning("No review image found, appending NA to it")
                self.review_image.append('NA')

        logging.info(f"Length of review title: {len(self.review_title)}")
        logging.info(f"Length of review description: {len(self.review_desc)}")
        logging.info(f"Length of review image: {len(self.review_image)}")

        if len(self.review_title) == len(self.review_desc) == len(self.review_image):
            self.data_dict["review_title"] = self.review_title
            self.data_dict["review_desc"] = self.review_desc
            self.data_dict["review_image"] = self.review_image
            logging.info("Completed!")
        else:
            raise Exception("Length of review title, description and image does not match!")


import folium
from app.tools import convert_adress_to_coordinates

def create_bars_map(current_location, bars_geo, path='app/templates/_map.html'):
    current_coordinates = convert_adress_to_coordinates(current_location)

    map = folium.Map(
        location=current_coordinates,
        zoom_start=12,
    )
    folium.Marker(
    location=current_coordinates,
    popup='Here I am',
    icon=folium.Icon(color='red', icon='check')
    ).add_to(map)

    tooltip = "Let's drink!"
    for bar in bars_geo:
        name = bar['title']
        folium.Marker(bar['coordinates'], popup=f'<i>{name}</i>', tooltip=tooltip).add_to(map)
    map.save(path)


import nsetools
from pprint import pprint
from dotenv import load_dotenv
import os
import requests
from twilio.rest import Client

load_dotenv("./.env")

API_KEY = os.environ.get("NEWS_API_KEY")
ACCOUNT_SID = "AC6f06d4b0637f1dc296d7bd49c759ae6e"
AUTH_TOKEN = os.environ.get("TWILIO_AUTH_KEY")
FROM=os.environ.get("FROM")
TO=os.environ.get("TO")

DRREDDY_STOCK = "DRREDDY"
RELIANCE_STOCK = "RELIANCE"
GODREJCP_STOCK = "GODREJCP"

WATCHLIST = [DRREDDY_STOCK, RELIANCE_STOCK, GODREJCP_STOCK]

def getStockdata(share_name):
    nse = nsetools.Nse()
    stock = nse.get_quote(share_name)

    previous_data = stock["open"] - stock["previousClose"]
    today_data = stock["closePrice"] - stock["open"]
    total_diff = stock["closePrice"] - stock["previousClose"]
    company_name = stock["companyName"]
    perc_drop = stock["closePrice"] * 5 / 100

    return (previous_data, today_data, total_diff, company_name, perc_drop)

def sendMessage(send_msg):
    client = Client(ACCOUNT_SID, AUTH_TOKEN)
    message = client.messages \
                    .create(
                        body=send_msg,
                        from_=FROM,
                        to=TO
                    )
    print(message.status)

def getNews(share_name, previous_data, today_data, total_diff, company_name, perc_drop):
    if ((previous_data < 0 and today_data < 0) or
        (previous_data > 0 and today_data > 0) and
        (abs(total_diff) - 5 <= perc_drop <= abs(total_diff) + 5)):

        parameters = {"q":company_name, "sortBy": "relevancy", "apiKey": API_KEY}
        articles = requests.get(url="https://newsapi.org/v2/everything", params=parameters)
        data = articles.json()["articles"]

        news_title = []
        news_url = []
        published_date = []
        for news in data:
            news_title.append(news["title"])
            news_url.append(news["url"])
            published_date.append(news["publishedAt"])

        send_msg = ""
        if total_diff > 0:
            send_msg += f"{share_name}:  {round(abs(total_diff), 2)} ⬆\n\n"
        else:
            send_msg += f"{share_name}:  {round(abs(total_diff), 2)} ⬇\n\n"

        for i in range(3):
            send_msg += "Title: "+news_title[i]+"\n"+"URL: "+news_url[i]+"\n"+"Dated: "+published_date[i][0:10]+"\n\n"

        sendMessage(send_msg = send_msg)

for share in WATCHLIST:
    previous_data, today_data, total_diff, company_name, perc_drop = getStockdata(share)
    getNews(share_name = share, previous_data=previous_data, today_data=today_data, total_diff=total_diff, company_name=company_name, perc_drop=perc_drop)


from ytmusicapi import YTMusic
import csv
import time

ytmusic = YTMusic('./src/headers_auth.json')
playlists = ytmusic.get_library_playlists()
lib_playlists = {}
playlist_ids = []

for i in playlists:
    playlist_ids.append(i['playlistId'])
    lib_playlists[i['title']] = i

# print(playlist_ids)
playlist = ytmusic.get_playlist(playlist_ids[2])
# print(playlist)

new_playlist = ytmusic.create_playlist('Imported Songs','Youtube API Test!!')

filename = "songs.csv"
rows = []
# reading csv file
with open(filename, 'r') as csvfile:
    # creating a csv reader object
    csvreader = csv.reader(csvfile)

    # extracting field names through first row
    fields = next(csvreader)

    # extracting each data row one by one
    for row in csvreader:
        rows.append(row[0])

failed = []
file = open("failed_songs.txt","w")
count = 0

for i in rows:
    count+=1
    print(count)
    # time.sleep(0.5)
    # print("sleeping")
    song_result = ytmusic.search(i,'songs')

    # print(len(song_result))
    if len(song_result) > 0:
        if i != song_result[0]['title']:
            video_result = ytmusic.search(i,'videos')
            if len(video_result) > 0:
                # print(video_result[0]['videoId'])
                ytmusic.add_playlist_items(new_playlist,[video_result[0]['videoId']])
            else:
                print("Failed Plus ONEEEEE")
                file.write(i + ' \n')
        else:
            # print(song_result[0]['videoId'])
            ytmusic.add_playlist_items(new_playlist,[song_result[0]['videoId']])
    else:
        print("Failed Plus ONEEEEE")
        file.write(i + ' \n')

file.close()


import socket
import selectors
import argparse

def recv_data(conn):
    data = conn.recv(1024)
    if data:
        print(data.decode())
    else:
        e_poll.unregister(conn)
        conn.close()

def acc_conn(p_server):
    conn, addr = p_server.accept()
    print("[*] Acception connection from %s:%d" % (addr[0], addr[1]))
    e_poll.register(conn, selectors.EVENT_READ, recv_data)

if __name__ == "__main__":
    parser = argparse.ArgumentParser(description='tcp server options')
    parser.add_argument('-p', '--port', default='22222',help='listen to port', dest='port')
    args = parser.parse_args()
    CONN_ADDR = ("127.0.0.1", int(args.port))
    server = socket.socket()
    server.bind(CONN_ADDR)
    server.listen(6)
    e_poll = selectors.DefaultSelector()
    e_poll.register(server, selectors.EVENT_READ, acc_conn)
    print("port: {}  wating for connect...".format(int(args.port)))
    while True:
        # 事件循环不断地调用select获取被激活的socket
        events = e_poll.select()
        # print(events)
        for key, mask in events:
            call_back = key.data
            # print(key.data)
            call_back(key.fileobj)


import math

import torch
import torch.nn as nn
import torch.nn.functional as F

class SingleDirectGNN(nn.Module):
    def __init__(self, hidden_size, pos_emb, step=1):
        super(SingleDirectGNN, self).__init__()
        self.pos_emb = pos_emb
        self.step = step
        self.hidden_size = hidden_size

        self.b_iah = nn.Parameter(torch.Tensor(self.hidden_size))
        self.b_oah = nn.Parameter(torch.Tensor(self.hidden_size))
        self.linear_edge_in = nn.Linear(self.hidden_size, 1 * self.hidden_size, bias=False)
        # self.linear_gate_i = nn.Linear(self.hidden_size, 3 * self.hidden_size, bias=False)
        # self.linear_gate_o = nn.Linear(self.hidden_size, 3 * self.hidden_size, bias=False)

        self.reset_parameters()

    def GNNCell(self, A_in, hidden):
        '''
        A: b x t x t
        '''
        h_n = self.linear_edge_in(hidden)
        # h_n = hidden
        # inputs_out = torch.matmul(A_out, h_n) + self.b_iah
        inputs_in = torch.matmul(A_in.transpose(1, 2), h_n) + self.b_oah
        return inputs_in

    def reset_parameters(self):
        stdv = 1.0 / math.sqrt(self.hidden_size)
        for weight in self.parameters():
            weight.data.uniform_(-stdv, stdv)

    def get_pos_emb(self, seq_mask):
        length = seq_mask.sum(1, keepdim=True)
        coo = torch.arange(seq_mask.size(1), device=seq_mask.device).unsqueeze(0).expand(seq_mask.size(0), -1)
        coo = coo * seq_mask
        pos_index = (length - coo) * seq_mask
        # pos_index = (coo + 1) * seq_mask
        return self.pos_emb(pos_index)

    def pos_graph(self, seq_inputs):
        seq_inputs = seq_inputs.unsqueeze(-1)  # b x t x 1
        seq_mask = seq_inputs.bool()  # b x t x 1

        relation_matrix = seq_inputs - seq_inputs.transpose(1, 2)  # b x t x t
        relation_matrix = relation_matrix.bool()  # b x t x t
        relation_matrix = (~relation_matrix).float().tril(diagonal=0)
        relation_matrix = relation_matrix * seq_mask * seq_mask.transpose(1, 2)

        # norm = relation_matrix.sum(-1, keepdim=True)  # b x t x 1
        # norm[torch.where(norm == 0)] = 1.
        # A_out = relation_matrix / norm

        norm = relation_matrix.sum(-2, keepdim=True)  # b x t x 1
        norm[torch.where(norm == 0)] = 1.
        A_in = relation_matrix / norm
        return A_in  # b x t x t

    def pos_dis(self, seq_mask):
        seq_mask = seq_mask.unsqueeze(-1).long()
        coo = torch.arange(seq_mask.size(-2), device=seq_mask.device).unsqueeze(0).unsqueeze(-1)  # 1 x t x 1
        res = coo - coo.transpose(1, 2)  # 1 x t x t
        res = res.tril(diagonal=0)
        return res.expand(seq_mask.size(0), -1, -1)

    def forward(self, seq_inputs, seq_ori_hidden=None):
        seq_mask = seq_inputs.bool()
        A_in = self.pos_graph(seq_inputs)

        pos_emb = self.get_pos_emb(seq_mask)
        pos_emb = F.normalize(pos_emb)  # b x t x h

        for i in range(self.step):
            pos_emb = self.GNNCell(A_in, pos_emb)

        return pos_emb  # b x t x h


#ncat listener to open a socket and get the words

from pyspark.sql import SparkSession
from pyspark.sql.functions import expr

if __name__ == "__main__":
    spark = SparkSession \
        .builder \
        .appName("streaming word count") \
        .config("spark.streaming.stopGracefullyOnShutdown","true") \
        .config("spark.sql.shuffle.partitions",3) \
        .getOrCreate()

    #read a streaming source - input dataframe
    #transform - output dataframe
    #write to sink

    #readStream is similar infunctionality as createDataFrame , it is a streaming dataframe some standard operations are not allowed (cannot apply count)
    lines_df = spark.readStream.format("socket").option("host","localhost").option("port","9999").load()

    lines_df.printSchema()

    words_df = lines_df.select(expr("explode(split(value,' ')) as word"))

    #count aggregate is allowed on streaming dataframe
    counts_Df = words_df.groupBy("word").count()

    #write dataframe to sink  - data stream writer is similar to data frame writer
    word_count_query = counts_Df.writeStream.format("console").option("checkpointLocation","chk-point-dir").outputMode("complete").start()
    #complete - it shows full output , the start() method starts the streaming

    #start- method starts a background job and return , so we must wait for the background job tom completed

    word_count_query.awaitTermination()


# This Python 3 environment comes with many helpful analytics libraries installed

# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python

# For example, here's several helpful packages to load in

import numpy as np # linear algebra

import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)

# Input data files are available in the "../input/" directory.

# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory

import os

for dirname, _, filenames in os.walk('/kaggle/input'):

    for filename in filenames:

        print(os.path.join(dirname, filename))

# Any results you write to the current directory are saved as output.
import matplotlib.pyplot as plt

import seaborn as sns

from keras.layers import Dense

from keras.layers import LSTM

from keras.layers import Dropout

from keras.models import Sequential

from keras.wrappers.scikit_learn import KerasRegressor

from sklearn.model_selection import GridSearchCV

from sklearn.metrics import mean_squared_log_error

from sklearn.preprocessing import StandardScaler
raw_train_data = pd.read_csv('/kaggle/input/covid19-global-forecasting-week-3/train.csv')

raw_test_data = pd.read_csv('/kaggle/input/covid19-global-forecasting-week-3/test.csv')

sub_data = pd.read_csv('/kaggle/input/covid19-global-forecasting-week-3/submission.csv')
# change dtypes, fillna

train_data = raw_train_data

train_data['Date'] = pd.to_datetime(train_data['Date'])

train_data['Province_State'] = train_data['Province_State'].fillna('None')

test_data = raw_test_data

test_data['Date'] = pd.to_datetime(test_data['Date'])
# unique identifiers for each province and country

ids = (

    train_data[['Province_State','Country_Region']]

    .drop_duplicates().reset_index(drop=True).to_dict('index')

)
# dict of dfs for each province

region_dfs_train = {i:train_data[(train_data['Province_State']==ids[i]['Province_State'])&

          (train_data['Country_Region']==ids[i]['Country_Region'])] for i in ids}
# function to convert series into supervised learning problem

def series_to_supervised(data, n_in=1, n_out=1, dropnan=True):

    """

    Frame a time series as a supervised learning dataset.

    Arguments:

        data: Sequence of observations as a list or NumPy array.

        n_in: Number of lag observations as input (X).

        n_out: Number of observations as output (y).

        dropnan: Boolean whether or not to drop rows with NaN values.

    Returns:

        Pandas DataFrame of series framed for supervised learning.

    """

    n_vars = 1 if type(data) is list else data.shape[1]

    df = pd.DataFrame(data)

    cols, names = list(), list()

    # input sequence (t-n, ... t-1)

    for i in range(n_in, 0, -1):

        cols.append(df.shift(i))

        names += [('var%d(t-%d)' % (j+1, i)) for j in range(n_vars)]

    # forecast sequence (t, t+1, ... t+n)

    for i in range(0, n_out):

        cols.append(df.shift(-i))

        if i == 0:

            names += [('var%d(t)' % (j+1)) for j in range(n_vars)]

        else:

            names += [('var%d(t+%d)' % (j+1, i)) for j in range(n_vars)]

     # put it all together

    agg = pd.concat(cols, axis=1)

    agg.columns = names

    # drop rows with NaN values

    if dropnan:

        agg.dropna(inplace=True)

    return agg
# prepare train and val data into sequences with n_in = lag time steps and n_out = future time steps

n_in = 6

n_out = 1

n_feats = 2 # cases and fatalities

train_supervised = [series_to_supervised(region_dfs_train[i][['ConfirmedCases','Fatalities']],

                                           n_in=n_in, n_out=n_out, dropnan=True) for i in region_dfs_train]

reshaped_train_data = pd.concat(train_supervised).reset_index(drop=True)
# drop 7000 zero sequences

zero_indices = reshaped_train_data[reshaped_train_data.sum(axis=1)==0].sample(8000).index

model_data = reshaped_train_data.drop(zero_indices,axis=0)
model_data.shape
# LSTM input is 3D (samples, timesteps, feats), output is 2D(samples,feats)

# cross val data

restack = reshaped_train_data.values

x = restack[:,:-2].reshape(len(restack),n_in,2)

y = restack[:,-2:]
print(x.shape,y.shape)
import keras.backend as K

def rmsle(pred,true):

    assert pred.shape[0]==true.shape[0]

    return K.sqrt(K.mean(K.square(K.log(pred+1) - K.log(true+1))))
def build_regressor(lstm_nodes,d1,d2):

    # define model

    model = Sequential()

    model.add(LSTM(lstm_nodes, activation='relu', input_shape=(n_in,2)))

    model.add(Dense(d1, activation='relu'))

    model.add(Dense(d2, activation='relu'))

    model.add(Dense(2, activation='relu'))

    model.compile(optimizer='adam', loss='mean_squared_logarithmic_error')

    return model
# scikit wrapper function

regressor = KerasRegressor(build_fn = build_regressor,verbose=0)

# grid search parameters

parameters = {'lstm_nodes':[12,40],

             'nb_epoch':[100],

             'batch_size':[32,256],

             'd1':[32,64],

             'd2':[16,32]}

gridsearch = GridSearchCV(estimator = regressor,

                 param_grid = parameters,

                 scoring = 'neg_mean_squared_log_error',

                 cv = 10,

                 n_jobs = -1,

                 verbose =0)
# restack train and val

gridsearch = gridsearch.fit(x,y)
gridsearch.cv_results_
gridsearch.cv_results_['mean_test_score']
gridsearch.cv_results_['std_test_score']
model = build_regressor(lstm_nodes=gridsearch.best_params_['lstm_nodes'],

                       d1=gridsearch.best_params_['d1'],

                       d2=gridsearch.best_params_['d2'])

model.save("model.h5")
history = model.fit(x, y, epochs=gridsearch.best_params_['nb_epoch'],

                    batch_size=gridsearch.best_params_['batch_size'])
# number of days to predict

pred_days = test_data['Date'].max()-test_data['Date'].min()

pred_days
# first batch of predictions

first_predict_date = test_data['Date'].min()

pred_data = {key:region_dfs_train[key].loc[(region_dfs_train[key]['Date']>=(first_predict_date-pd.DateOffset(days=n_in)))&

                                    (region_dfs_train[key]['Date']<first_predict_date)]

             for key in region_dfs_train}

test_reshaped = [pred_data[i][['ConfirmedCases','Fatalities']].values.reshape(1,n_in,2) for i in pred_data]

first_input = np.vstack(test_reshaped)

first_pred = model.predict(first_input)
# iterate prediction output back into model input for the next days

pin = [first_input]

pout = [first_pred]

# first prediction is done outside of loop, need to loop for following 41 days

for i in range(42):

    p = model.predict(pin[i])

    pout.append(p)

    t= np.insert(pin[i],n_in,pout[i],axis=1)[:,1:,:]

    pin.append(t)
# create the prediction dataframe

pred_df = pd.DataFrame(np.concatenate(pout))

pred_df.columns = ['ConfirmedCases','Fatalities']

pred_df['Date'] = np.repeat(test_data['Date'].unique(),len(ids))

pred_df['Province_State'] = list(test_data.drop_duplicates(subset=['Province_State','Country_Region'])['Province_State'])*43

pred_df['Country_Region'] = list(test_data.drop_duplicates(subset=['Province_State','Country_Region'])['Country_Region'])*43

pred_df = pred_df.sort_values(by=['Country_Region','Date']).reset_index(drop=True)
def rmsle_check(pred,true):

    p = np.log(pred+1)

    a = np.log(true+1)

    s = np.sum((p-a)**2)

    return np.sqrt((1/len(pred))*s)
true = train_data[train_data['Date']>=test_data['Date'].min()]

pred = pred_df[pred_df['Date']<=train_data['Date'].max()]

rmsle_check(pred['ConfirmedCases'],true['ConfirmedCases']),rmsle_check(pred['Fatalities'],true['Fatalities'])
sub = pred_df[['ConfirmedCases','Fatalities']]

sub['ForecastId'] = test_data['ForecastId']
sub.to_csv("submission.csv",index=False)


# This Python 3 environment comes with many helpful analytics libraries install

# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python

# For example, here's several helpful packages to load in

import numpy as np # linear algebra

import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)

# Input data files are available in the "../input/" directory.

# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory

import os

print(os.listdir("../input"))

# Any results you write to the current directory are saved as output.

# Importing all of Keras Modules

import keras

from keras import models

from keras import layers

from keras.layers import LeakyReLU

from keras.layers import ELU

from keras.callbacks import ModelCheckpoint, ReduceLROnPlateau, EarlyStopping

import matplotlib.pyplot as plt

from sklearn.utils import shuffle

from sklearn.model_selection import train_test_split
# Reading the train and test data

train = pd.read_csv("../input/train.csv")

test = pd.read_csv("../input/test.csv")
# Creating a copy of the training data set to conduct EDA:

train_copy = train.copy()

test_copy = test.copy()
# The shape of the data sets are:

print("training data contains ", train_copy.shape[0]," records and ",train_copy.shape[1], " columns \n")

print("training data contains ", test.shape[0]," records and ",test.shape[1], " columns \n")
# Taking a look at the training data

train_copy.info()
# Collecting summary statistics for all the columns:

train_copy.describe()
#columns = list(train_copy)

#for i in columns:

#    if train_copy[i].dtype != np.int64 and train_copy[i].dtype != np.float64:

#        print(train_copy[i].value_counts())

# Dropping the sparse columns, ID (from both Trainig and test)

train_copy.drop(["Id","Alley","FireplaceQu","PoolQC","Fence","MiscFeature"], axis = 1, inplace=True)

test_copy.drop(["Id","Alley","FireplaceQu","PoolQC","Fence","MiscFeature"], axis = 1, inplace=True)
train_copy.head()
test_copy.head()
# Checking for Ordinal Variables

columns = list(train_copy)

for i in columns:

    if train_copy[i].dtype != np.int64 and train_copy[i].dtype != np.float64:

        print(train_copy.groupby(i)['SalePrice'].agg({"average_price":"mean", "#Entries":"count"}).sort_values(by=['average_price']))
# Missing value treatment. Replacing String Columns with 'NAs' and Numerical Columns with '-99' (junk value)

#Training data

columns_train = list(train_copy)

for i in columns_train:

    if train_copy[i].dtype == 'object':

        train_copy[i] = train_copy[i].fillna('Na')

    else:

        train_copy[i] = train_copy[i].fillna(-99)
# Missing value treatment. Replacing String Columns with 'NAs' and Numerical Columns with '-99' (junk value)

# Test data

columns_test = list(test_copy)

for i in columns_test:

    if test_copy[i].dtype == 'object':

        test_copy[i] = test_copy[i].fillna('Na')

    else:

        test_copy[i] = test_copy[i].fillna(-99)
# Seperating examples and labels from the training data set

X =  train_copy.drop(['SalePrice'], axis=1)

y = np.log(train_copy.SalePrice)
X.head()
# Doing one hot encoding for both training and test data sets

X = pd.get_dummies(X)

test_copy = pd.get_dummies(test_copy)
print("one hot encoded training data set shape", X.shape, "\n")

print("one hot encoded training data set shape", test_copy.shape, "\n")
common_cols = [a for a in list(train_copy) for b in list(test_copy) if a==b]

len(common_cols)
# Subsetting both train and test data sets for the common columns:

train_copy_comm = X[common_cols]

test_copy_comm = test_copy[common_cols]
# Standardizing the training and test data set

mean = train_copy_comm.mean(axis = 0)

std = train_copy_comm.std(axis = 0)

train_copy_comm -= mean

train_copy_comm /= std

test_copy_comm -= mean

test_copy_comm /= std
# The above operation has introduced NAs in the data set becuase of the division operation. We need to do the null value imputation again

train_copy_comm = train_copy_comm.fillna(0)

test_copy_comm = test_copy_comm.fillna(0)
train_copy_comm.head()
train_data, validation_data, train_targets, validation_targets = train_test_split(train_copy_comm, y,test_size=0.2)
# Building the network

# Model Build

from keras import backend as K

def root_mean_squared_error(y_true, y_pred):

        return K.sqrt(K.mean(K.square(y_pred-y_true)))

def build_model_dropout():

    model = models.Sequential()

    model.add(layers.Dense(240,activation='relu',

                           input_shape=(train_copy_comm.shape[1],)))

    model.add(layers.Dropout(0.45))

    model.add(layers.Dense(240,activation='relu'))

    model.add(layers.Dropout(0.45))

    model.add(layers.Dense(240,activation='relu'))

    model.add(layers.Dropout(0.45))

    model.add(layers.Dense(120,activation='relu'))

    model.add(layers.Dropout(0.35))

    model.add(layers.Dense(120,activation='relu'))

    model.add(layers.Dropout(0.35))

    model.add(layers.Dense(60,activation='relu'))

    model.add(layers.Dropout(0.2))

    model.add(layers.Dense(60,activation='relu'))

    model.add(layers.Dense(1))

    model.compile(optimizer='rmsprop', loss=root_mean_squared_error, metrics=['mse'])

    model.summary()

    return model
# Trying out iterated K-Fold Validation with Shuffling

num_epochs = 900

all_mse_histories = []

# Invoking the model function

model = build_model_dropout()

#Creating a checkpoint for the network

filepath="weights.best.hdf5"

checkpoint = ModelCheckpoint(filepath, monitor='val_loss', verbose=1, save_best_only=True, mode='min')

#Defining a callback function to reduce the learning rate near the minim

reduce_learning_rate = ReduceLROnPlateau(monitor = "val_loss", factor = 0.9,patience=50, min_lr=0.001,  verbose=1)

#Defining a callback function for early stopping

early_stopping = EarlyStopping(monitor='val_loss', mode='min', verbose=1, patience=50)

#All the callbacks list

callbacks_list = [checkpoint,reduce_learning_rate,early_stopping]

history = model.fit(train_data, train_targets, validation_data=(validation_data,validation_targets), epochs=num_epochs,

                callbacks=callbacks_list, batch_size=20,verbose=2)

mse_history = history.history['val_mean_squared_error']

all_mse_histories.append(mse_history)

average_mse_history = history.history['val_mean_squared_error']
def smooth_curve(points, factor=0.9):

    smoothed_points = []

    for point in points:

        if smoothed_points:

            previous = smoothed_points[-1]

            smoothed_points.append(previous*factor + point*(1-factor))

        else:

            smoothed_points.append(point)

    return smoothed_points
smooth_mse_history = smooth_curve(average_mse_history)

plt.plot(range(1, len(smooth_mse_history) + 1), smooth_mse_history)

plt.xlabel('Epochs')

plt.ylabel('Validation MAE')

plt.show()
# load the model

from keras.models import Sequential, load_model

new_model = load_model("weights.best.hdf5", custom_objects={'root_mean_squared_error': root_mean_squared_error})
# estimate accuracy on whole dataset using loaded weights

scores = new_model.evaluate(validation_data, validation_targets, verbose=0)

print("%s: %.4f%%" % (new_model.metrics_names[1], scores[1]))

print("%s: %.4f%%" % (new_model.metrics_names[0], scores[0]))

#0.1330
predictions = new_model.predict(test_copy_comm)

submission = pd.read_csv('../input/sample_submission.csv')

submission.SalePrice = np.exp(predictions)

submission.to_csv('submission.csv', index=False)


import argparse
import socket

def scan_port(host, port):
    # Create a socket object
    sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
    sock.settimeout(1)

    # Connect to the server
    result = sock.connect_ex((host, port))

    # Close the socket
    sock.close()

    return result == 0

# Parse command-line arguments
parser = argparse.ArgumentParser(description="Port scanner")
parser.add_argument("-host", help="the host to scan")
parser.add_argument("-p", "--ports", metavar="port_range", type=str,
                    help="the range of ports to scan (e.g. 1-1024)",
                    default="1-1024")
args = parser.parse_args()

# Parse port range
start_port, end_port = map(int, args.ports.split("-"))
ports = range(start_port, end_port + 1)

# Scan each port
for port in ports:
    if scan_port(args.host, port):
        print(f"Port {port} is open")
    else:
        print(f"Port {port} is closed")


"""empty message

Revision ID: 757b49c77662
Revises: 6af3389a2870
Create Date: 2018-12-03 11:40:36.775188

"""
from alembic import op
import sqlalchemy as sa

# revision identifiers, used by Alembic.
revision = '757b49c77662'
down_revision = '6af3389a2870'
branch_labels = None
depends_on = None

def upgrade():
    # ### commands auto generated by Alembic - please adjust! ###
    op.create_index(op.f('ix_business_id'), 'business', ['id'], unique=False)
    # ### end Alembic commands ###

def downgrade():
    # ### commands auto generated by Alembic - please adjust! ###
    op.drop_index(op.f('ix_business_id'), table_name='business')
    # ### end Alembic commands ###


from models.rnn import Zhang
import utils.functions as f

import numpy as np
import cupy as cp
import matplotlib.pyplot as plt
from models.runner import test
from utils.visualize import calc_ratemap, plot_ratemap

dim = 1
B = 100
T = 1000
joystick = False
visualize = True
# env and agent
if dim == 1:
    from envs.env import Env
    env = Env(batch=B, dim=dim, d=2 * np.pi, periodic_boundary=True, random_init=False)
    from envs.agent import GaussMomentumAgent

    # agent = ConstAgent(batch=B, dim=dim, mu=1.)
    agent = GaussMomentumAgent(batch=B, dim=dim, sigma=1.2, momentum=0.8)
elif dim == 2:
    from envs.arena import ArenaEnv2D
    env = ArenaEnv2D(batch=B, random_init=False)
    if joystick is False:
        from envs.agent import RayleighAgent2D
        agent = RayleighAgent2D(batch=B, mu=0.2, sigma=3.)
    else:
        from cns.envs.agent import JoyStickAgent2D
        agent = JoyStickAgent2D(batch=B)

# hyperparamters
if dim == 1:
    norm = False
    alpha = 1.0  # dt / tau
    n = 100
    N = n**dim

    a = 0.02
    b = None

    D, Dx = f.get_1D(n, require_Dx=True)
    c = 0.1
    k = 0.06
    h = -0.03
    W = c * np.cos(k * D) + h
    Wx = -c * np.sin(k * D) * k * Dx
    Wv = a * np.stack((Wx,), axis=0)
elif dim == 2:
    norm = False
    alpha = 0.2
    n = 32
    N = n**dim

    A = 5.
    lbd = 13.
    abar = 1.05
    beta = 3. / lbd**2
    a = 1.0
    b = 1.0 * np.ones(N)

    D, Dx, Dy = f.get_2D(n, require_Dx=True)
    W = f.dog(D, A, abar * beta, A, beta)
    Wx = f.dog_grad(D, Dx, A, abar * beta, A, beta)
    Wy = f.dog_grad(D, Dy, A, abar * beta, A, beta)
    Wv = a * np.stack((Wx, Wy), axis=0)

# model
model = Zhang(in_size=dim, hid_size=N, nonlinear='relu-tanh', alpha=alpha, bias=True, norm=norm)
model.set_weights(W=W, b=b, W_in=Wv)
model.to_gpu()

# init: you have to control the initial conditions across batches
np.random.seed(0)
u = np.tile(np.random.normal(loc=0., scale=1. / np.sqrt(N), size=N), (B, 1)).astype(np.float32)
u = cp.asarray(u)

# test
rs, xs, vs, x0 = test(env=env, agent=agent, model=model, u=u, T=T, T_init=500, visualize=visualize, joystick=joystick)
print(rs.shape, xs.shape)
np.savez('./data/dim={}_{}_B={}_T={}_N={}_noise=0.1'.format(dim, norm, B, T, N), rs=rs, xs=xs, vs=vs, x0=x0)

# ratemap
if dim == 1:
    vxs = np.concatenate((vs, xs), axis=-1)
    activations = calc_ratemap(r=rs, x=vxs, coords=((-5, 5), (0, 2 * np.pi)), resolutions=(4, 10))
elif dim == 2:
    activations = calc_ratemap(r=rs[:, :, 0:32], x=xs, coords=((-1.1, 1.1), (-1.1, 1.1)), resolutions=(10, 10))
rm_fig = plot_ratemap(activations, n_plots=len(activations), width=10)
plt.savefig("{}.png".format(dim))


"""Upload img in Blog

Revision ID: c2c7704dbc13
Revises: d1add5e3e65e
Create Date: 2017-06-27 10:41:26.856029

"""

# revision identifiers, used by Alembic.
revision = 'c2c7704dbc13'
down_revision = 'd1add5e3e65e'

from alembic import op
import sqlalchemy as sa
from sqlalchemy.dialects.postgresql import JSON

field = 'media_url'

def upgrade():
    op.add_column('blogpost', sa.Column(field, sa.String))
    op.add_column('blogpost', sa.Column('info', JSON))

def downgrade():
    op.drop_column('blogpost', field)
    op.drop_column('blogpost', 'info')


from helpers.http_wrapper import https_get
from helpers.payload import Payload
import json
import time
import logging

URL_TEMPLATE = "https://graph.microsoft.com/v1.0/me/drive/root:{0}:/children"

RETRY_WAIT = 5 #In case of potentially recoverable error the time lapse between tries

## This will list all the files
def listfiles(path, headers):
    logging.debug("listfiles(): Starting list files operation...")

    uri = URL_TEMPLATE.format(path)

    logging.debug("listfiles(): Listing Remote Path: %s",path)

    page_number = 0
    retry_count=0

    item_list = list()

    ## Loop until there are further pages
    while(True):

        response = https_get(uri, headers=headers)

        # Used to simulate very rare error
        #if(page_number == 1 and retry_count == 0):
        #    response = Payload(503, "Simulation Error","")

        logging.debug("listfiles(): Status: {} and reason: {}".format(response.get_status_code(), response.get_status_message()))

        ## Sucessfull response - Lets process it
        if(response.get_status_code() == 200):
            retry_count=0
            page_number = page_number + 1

            data_parsed = json.loads(response.get_payload())

            logging.debug("listfiles() - Page Number: %s", page_number)
            number_items = data_parsed["@odata.count"]
            logging.debug("listfiles() - Total Items: %s", number_items)

            next_link = None

            if('@odata.nextLink' in data_parsed.keys()):
                next_link = data_parsed["@odata.nextLink"]
                logging.debug("listfiles() - THERE ARE ADDITIONAL PAGES. Next link: %s", next_link)

            else:
                logging.debug("listfiles() - no further pages")
                next_link = None

            ## Mapping just the relevant fields
            for item in data_parsed["value"]:
                created_date_time = item["createdDateTime"]
                last_modified_datetime = item["lastModifiedDateTime"]
                ctag = item["cTag"]
                etag = item["eTag"]
                item_id = item["id"]
                name = item["name"]
                item_type = ""  # folder/file
                size = 0 #only return size for files

                #Determine item type for the listing
                if("folder" in item):
                    item_type = "folder"

                if("file" in item):
                    item_type = "file"
                    size = item["size"]

                item_data= { "type": item_type,
                            "id": item_id,
                            "name": name,
                            "size": size,
                            "ctag": ctag,
                            "etag": etag,
                            "created": created_date_time,
                            "lastmodified": last_modified_datetime
                }
                item_list.append(item_data)

            logging.debug("listfiles(): Page completed: %s", page_number)

            #Check if we exit the paging when it ends or not
            if(next_link == None):
                logging.debug("listfiles(): Total number of items returned: %s", len(item_list))
                return { "status":True,
                        "message":"Success",
                        "itemlist":item_list
                }
            else:
                uri = next_link #Prepare the URL for the next page

        # Status: 503 and reason: Service Unavailable
        # This is a recoverable error.
        elif(response.get_status_code() == 503):
            retry_count = retry_count + 1
            time.sleep(RETRY_WAIT) #Sleep 5 seconds before retry
            logging.warn("listfiles(): 503 Error - Sleeping 5 seconds and retrying")

        ## The specified path does not exist
        elif(response.get_status_code() == 404):
            return {
                    "status": False,
                    "message":"Path not found."
                }

        elif(response.get_status_code() == 401):
            return {
                    "status": False,
                    "message":"Unauthorized."
                }

        # There were exceptions on the http which exceeded the interval between retried and we had to abort.
        elif(response.get_status_code() == 0):
            return {
                    "status": False,
                    "message":response.get_status_message()
                }
        else:
            return {
                    "status": False,
                    "message": response.get_status_message()
                }

        if(retry_count > 3):
            return {
                    "status": False,
                    "message":"Failure after number of retries exceeded"
                }


import argparse

parser = argparse.ArgumentParser()

parser.add_argument('--input_pdb', metavar='PATH', default=None,
                    help='initialize first 2 GNN layers with pretrained weights')
parser.add_argument('--output_pt', metavar='PATH', default=None,
                    help='evaluate a trained model')
args = parser.parse_args()

from pyrosetta import *

import numpy as np

np.set_printoptions(threshold=np.inf)
init_cmd = list()
init_cmd.append("-multithreading:interaction_graph_threads 1 -multithreading:total_threads 1")
init_cmd.append("-hb_cen_soft")
init_cmd.append("-detect_disulf -detect_disulf_tolerance 2.0")
init_cmd.append("-relax:dualspace true -relax::minimize_bond_angles -default_max_cycles 200")
init_cmd.append("-mute all")
init_cmd.append("-constant_seed")
init_cmd.append("-read_only_ATOM_entries")
init(" ".join(init_cmd))

def extractSS(pose):
    dssp = rosetta.core.scoring.dssp.Dssp(pose)
    dssp.insert_ss_into_pose(pose)
    fp = open(args.output_pt, 'w')
    for ires in range(1, pose.size()+1):
        SS = pose.secstruct(ires)
        print(str(ires)+'\t'+SS, file=fp)

def main():
    init_pose=pose_from_pdb(args.input_pdb)
    extractSS(init_pose)

if __name__== "__main__":
    main()


import requests
from flask import Flask, render_template, request
from bs4 import BeautifulSoup

"""
When you try to scrape reddit make sure to send the 'headers' on your request.
Reddit blocks scrappers so we have to include these headers to make reddit think
that we are a normal computer and not a python script.
How to use: requests.get(url, headers=headers)
"""

headers = {'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/35.0.1916.47 Safari/537.36'}

"""
All subreddits have the same url:
i.e : https://reddit.com/r/javascript
You can add more subreddits to the list, just make sure they exist.
To make a request, use this url:
https://www.reddit.com/r/{subreddit}/top/?t=month
This will give you the top posts in per month.
"""

db = {}
reddit_url="https://www.reddit.com"
post_cap=6
subreddits = [
    "javascript",
    "reactjs",
    "reactnative",
    "programming",
    "css",
    "golang",
    "flutter",
    "rust",
    "django",
    "pics"
]

def get_data(subreddit):
  if db.get(subreddit) is None:
    req=requests.get(f"{reddit_url}/r/{subreddit}/top/?t=month", headers=headers)
    soup=BeautifulSoup(req.text,"html.parser")
    reddit_card=soup.find_all("div",{"class":"scrollerItem"},limit=post_cap)
    print(f"extract {subreddit}")
    data_list=[]
    for card in reddit_card:
      #remove ad(promoted post)
      try:
        link=reddit_url+card.find("a",{"data-click-id":"body"})["href"]
      except:
        continue
      ele={
        "upvote":card.find("div").text,
        "title":card.find("h3").text,
        "link":link,
        "keyward":f"r/{subreddit}"
      }
      data_list.append(ele)
    db[subreddit]=data_list
  return db[subreddit]

def number(upvote):
  try:
    vote=int(upvote)
  except:
    if upvote[-1] == 'k':
      vote=int(float(upvote[:-2])*1000)
    elif  upvote[-1] == 'm':
      vote=int(float(upvote[:-2])*1000000)
    else:
      vote=0
  return vote

app = Flask("DayEleven")

@app.route('/')
def home():
  return render_template("home.html",subreddits=subreddits)

@app.route('/read')
def read():
  data_list=[]
  keywards=""
  for subreddit in subreddits:
    is_on=request.args.get(subreddit)
    if is_on == "on":
      data_list.extend(get_data(subreddit))
      keywards+=f" r/{subreddit}"
  data_list=sorted(data_list,reverse=True,key=lambda item:number(item['upvote']))
  return render_template("read.html",keywards=keywards,data_list=data_list)

app.run(host="0.0.0.0")


from __future__ import annotations

from collections.abc import Callable
from dataclasses import dataclass
import logging
from typing import Final

from homeassistant.components.binary_sensor import (
    BinarySensorDeviceClass,
    BinarySensorEntity,
    BinarySensorEntityDescription,
)
from homeassistant.config_entries import ConfigEntry
from homeassistant.core import HomeAssistant
from homeassistant.helpers.entity_platform import AddEntitiesCallback

from .const import DOMAIN
from .coordinator import MapShareCoordinator
from .entity import MapShareBaseEntity

_LOGGER = logging.getLogger(__name__)

@dataclass
class MapShareBinarySensorEntityDescription(BinarySensorEntityDescription):
    """A class that describes custom binary sensor entities."""

    is_on: Callable[[dict[str, str]], bool] | None = None
    on_icon: str | None = None
    off_icon: str | None = None

SENSOR_DESCRIPTIONS: Final[tuple[MapShareBinarySensorEntityDescription, ...]] = (
    MapShareBinarySensorEntityDescription(
        key="in_emergency",
        name="In Emergency",
        is_on=lambda raw_values: raw_values.get('In Emergency', False) == "True",
        on_icon="mdi:alert-circle",
        off_icon="mdi:alert-circle",
        device_class=BinarySensorDeviceClass.PROBLEM,
    ),
    MapShareBinarySensorEntityDescription(
        key="valid_gps_fix",
        name="Valid GPS Fix",
        is_on=lambda raw_values: raw_values.get('Valid GPS Fix', False) == "True",
        on_icon="mdi:crosshairs-gps",
        off_icon="mdi:crosshairs",
    ),
)

async def async_setup_entry(
    hass: HomeAssistant,
    config_entry: ConfigEntry,
    async_add_entities: AddEntitiesCallback,
) -> None:
    """Set up binary_sensor platform."""
    coordinator: MapShareCoordinator = hass.data[DOMAIN][config_entry.entry_id]
    entities: list[MapShareConnectBinarySensor] = []
    for description in SENSOR_DESCRIPTIONS:
        entities.append(
            MapShareConnectBinarySensor(coordinator, description)
        )
    async_add_entities(entities)
    return True

class MapShareConnectBinarySensor(BinarySensorEntity, MapShareBaseEntity):

    def __init__(
        self,
        coordinator: MapShareCoordinator,
        description: MapShareBinarySensorEntityDescription,
    ) -> None:
        """Initialize the sensor."""
        super().__init__(coordinator)
        self.entity_description: MapShareBinarySensorEntityDescription = description
        self._attr_unique_id = f"{coordinator.map_link_name}-{description.key}"
        self._attr_name = f"{description.name}"

    @property
    def is_on(self) -> bool | None:
        """Return true if the binary sensor is on."""
        if self.entity_description.is_on is not None:
            return self.entity_description.is_on(self.coordinator.raw_values)
        return None

    @property
    def icon(self):
        """Return the icon to use in the frontend, if any."""
        if (
            self.entity_description.on_icon == self.entity_description.off_icon
        ) is None:
            return BinarySensorEntity.icon
        return (
            self.entity_description.on_icon
            if self.is_on
            else self.entity_description.off_icon
        )


import platform
from configparser import ConfigParser
import os
from os.path import dirname, abspath

# Settings
parser = ConfigParser()
fileConfig = os.path.join(dirname(abspath(__file__)), "settings.ini")
parser.read(fileConfig)

ffmpegConf = parser["GENERAL"]["ffmpeg"]
DemConf = parser["GENERAL"]["DTM_file"]
min_buffer_size = int(parser["GENERAL"]["min_buffer_size"])

Platform_lyr = parser["LAYERS"]["Platform_lyr"]
Beams_lyr = parser["LAYERS"]["Beams_lyr"]
Footprint_lyr = parser["LAYERS"]["Footprint_lyr"]
FrameCenter_lyr = parser["LAYERS"]["FrameCenter_lyr"]
FrameAxis_lyr = parser["LAYERS"]["FrameAxis_lyr"]
Point_lyr = parser["LAYERS"]["Point_lyr"]
Line_lyr = parser["LAYERS"]["Line_lyr"]
Polygon_lyr = parser["LAYERS"]["Polygon_lyr"]
frames_g = parser["LAYERS"]["frames_g"]
Trajectory_lyr = parser["LAYERS"]["Trajectory_lyr"]
epsg = parser["LAYERS"]["epsg"]
Reverse_geocoding_url = parser["GENERAL"]["Reverse_geocoding_url"]
dtm_buffer = int(parser["GENERAL"]["DTM_buffer_size"])

# Geo variables
EARTH_MEAN_RADIUS = 6371008.8
WGS84String = "WGS84"
encoding = "utf-8"
defaultTargetWidth = 200.0

# Klv variables
UASLocalMetadataSet = b"\x06\x0e+4\x02\x0b\x01\x01\x0e\x01\x03\x01\x01\x00\x00\x00"
KlvHeaderKeyOther = b"\x06\x0e+4\x02\x01\x01\x01\x0e\x01\x01\x02\x01\x01\x00\x00"

# OS variables
isWindows = platform.system() == "Windows"

if isWindows:
    ffmpeg_path = os.path.join(ffmpegConf, "ffmpeg.exe")
    ffprobe_path = os.path.join(ffmpegConf, "ffprobe.exe")
else:
    ffmpeg_path = os.path.join(ffmpegConf, "ffmpeg")
    ffprobe_path = os.path.join(ffmpegConf, "ffprobe")


import tabulator
import json
import re
from pathlib import Path

fp = re.compile('\w+', re.UNICODE)

def fingerprint(x):
    return ''.join(fp.findall(x.upper()))

if __name__ == '__main__':
    # How to get places.csv from a nominatim server?
    # # kubectl exec -it nominatim-5d6889fd59-lxps7 -- su nominatim -c "psql -c \"\\copy (select hstore_to_json(name) from place where type in ('village', 'town', 'city', 'locality')) to stdout with csv\"" > places.csv
    # How to get official list of towns?
    # # https://data.gov.il/dataset/citiesandsettelments
    langs = {}
    s = tabulator.Stream('tools/yeshuvim.csv', headers=1)
    s.open()
    for item in s.iter(keyed=True):
        langs.setdefault('he', []).append(item['שם_ישוב'].strip().replace('(', 'XXX').replace(')', '(').replace('XXX', ')').replace('  ', ' '))
        langs.setdefault('en', []).append(item['שם_ישוב_לועזי'].strip().title().replace('  ', ' '))

    s = tabulator.Stream('tools/places.csv')
    s.open()
    for item in s.iter():
        if len(item) == 0: continue
        item = json.loads(item[0])
        for k, v in item.items():
            if k.startswith('name'):
                langs.setdefault(k[5:], []).append(v.strip(', '))

    combinations = {
        'he': ('he', ''),
        'ar': ('ar', 'he', ''),
        'en': ('en', 'he', ''),
        'es': ('es', 'en', ''),
        'fr': ('fr', 'en', ''),
        'ru': ('ru', 'en', ''),
    }
    suggestions = {}
    for k, v in combinations.items():
        print('COMBINATION', k)
        s = []
        for lang in v:
            for w in langs[lang]:
                if w:
                    fw = fingerprint(w)
                    found = False
                    for ww in s:
                        if ww == w:
                            found = True
                            break
                        if fingerprint(ww) == fw:
                            found = True
                            print('Matched %s based on fingerprint (kept %s)' % (w, ww))
                            break
                    if not found:
                        s.append(w)
        suggestions[k] = s

    for k, v in suggestions.items():
        print(k, len(v), v[:10])

    f_out = Path('src/app/city-suggestions.ts')
    f_out.open('w').write('''
/* tslint:disable */
export const citySuggestions = {};
'''.format(json.dumps(
        suggestions, ensure_ascii=False, sort_keys=True, indent=2)
        )
    )


import sys
import paho.mqtt.client as mqtt
import time
import threading
from Crypto.Cipher import AES
import base64

# https://www.itread01.com/content/1547711886.html
class prpcrypt():
    def __init__(self, key):
        self.key = key
        self.mode = AES.MODE_CBC

    def encrypt(self, text):
        iv = '0102030405060708'
        pad = lambda s: s + (16 - len(s)%16) * chr(16 - len(s)%16)
        text = pad(text)

        cipher = AES.new(self.key.encode('utf8'), self.mode, iv.encode('utf8'))
        encryptedbytes = cipher.encrypt(text.encode('utf8'))

        encodestrs = base64.b64encode(encryptedbytes)
        enctext = encodestrs.decode('utf8')
        return enctext

    def decrypt(self, text):
        iv = '0102030405060708'
        text = text.encode('utf8')
        encodebytes = base64.decodebytes(text)

        cipher = AES.new(self.key.encode('utf8'), self.mode, iv.encode('utf8'))
        text_decrypted = cipher.decrypt(encodebytes)
        unpad = lambda s: s[0:-s[-1]]
        text_decrypted = unpad(text_decrypted)
        text_decrypted = text_decrypted.decode('utf8')
        return text_decrypted

def main():
    MQTTTopicServerIP = "192.168.137.219"
    MQTTTopicServerPort = 1883
    MQTTTopicName = "wsn_lab/test"
    pc = prpcrypt(key = '0000000107062533')

    Message = input('>>')

    EncryptMessage = pc.encrypt(str(Message))
    # d = pc.decrypt(EncryptMessage)

    mqttc = mqtt.Client("python_pub")
    mqttc.connect(MQTTTopicServerIP, MQTTTopicServerPort)
    print(EncryptMessage)
    mqttc.publish(MQTTTopicName, EncryptMessage)

if __name__ == '__main__': main()


import warnings
import numpy
from ..utilities.extractParams import *
from datetime import datetime

def getSampleMetadataFromWatersRawFiles(rawDataPath):
	"""
	Get acquisition metadata from Waters RAW files and returns them as a dataframe

	:param str rawDataPath: Path to folder of raw data
	"""

	# Get the paramters as a table
	instrumentParams = extractParams(rawDataPath, 'Waters .raw')

	# Strip any whitespace from 'Sample File Name'
	instrumentParams['Sample File Name'] = instrumentParams['Sample File Name'].str.strip()

	# Parse acqustion times
	instrumentParams['Acquired Time'] = numpy.nan
	for i in range(instrumentParams.shape[0]):
		try:
			instrumentParams.loc[i, 'Acquired Time'] = datetime.strptime(str(instrumentParams.loc[i, '$$ Acquired Date:']) + " " + str(instrumentParams.loc[i,'$$ Acquired Time:']), '%d-%b-%Y %H:%M:%S')
		except ValueError:
			pass

	# Rename '$$ Acquired Time' and '$$ Acquired Date to avoid confusion
	instrumentParams.rename(columns={'$$ Acquired Time:': 'Measurement Time'}, inplace=True)
	instrumentParams.rename(columns={'$$ Acquired Date:': 'Measurement Date'}, inplace=True)

	##
	# Detect duplicate experiment filenames
	##
	duplicateSamples = instrumentParams.loc[instrumentParams['Sample File Name'].duplicated(keep=False)]
	if duplicateSamples.size > 0:
		warnings.warn('Duplicate raw data loaded, discarding duplicates.', UserWarning)
		# Drop duplicate files
		instrumentParams = instrumentParams.loc[instrumentParams['Sample File Name'].duplicated(keep='first')==False]

	return instrumentParams


from django.db import models
from django.utils.translation import gettext_lazy as _
from django.contrib.auth.models import AbstractUser

# custom user model

class User(AbstractUser):
    first_name = models.CharField(
        _("first name"),
        max_length=30,
    )
    last_name = models.CharField(
        _("last name"),
        max_length=150,
    )
    username = models.EmailField(
        _("email address"),
        unique=True,
        error_messages={
            "unique": _("A user with that username already exists."),
        },
    )

    EMAIL_FIELD = "username"

    def __str__(self):
        return self.username

class Child_care_facility(models.Model):
    name = models.CharField("Nom de la structure", max_length=50, unique=True)
    max_child_number = models.IntegerField(
        "Places maximum",
    )
    type_of_facility = models.CharField(
        "Type de structure",
        max_length=50,
        choices=[
            ("MC", "Micro-Crèche"),
            ("C", "Crèche"),
            ("MAM", "MAM"),
            ("MC", "Crèche-Parentale"),
            ("MC", "Crèche-Municipale"),
            ("MC", "Crèche d'entreprise"),
            ("MC", "Crèche associative"),
        ],
    )
    status = models.CharField(
        "Statut",
        max_length=50,
        choices=[
            ("A", "Active"),
            ("NA", "Non Active"),
            ("EC", "En cours de création"),
        ],
    )
    address = models.ForeignKey(
        "auth_access_admin.Address",
        on_delete=models.CASCADE,
        verbose_name="Adresse",
    )
    phone = models.CharField("Téléphone", max_length=14)
    email = models.EmailField("Adresse Email")

    class Meta:
        verbose_name = "Structure de Garde"
        verbose_name_plural = "Structures de Garde"

    def __str__(self):
        return self.name

class New(models.Model):
    date_time = models.DateTimeField(
        "Horodatage",
        auto_now_add=True,
    )
    title = models.CharField(
        "Titre",
        max_length=50,
    )
    content = models.CharField(
        "Contenu texte",
        max_length=200,
        blank=True,
    )
    img_url = models.ImageField("image", upload_to="news_img")
    cc_facility = models.ForeignKey(
        Child_care_facility,
        on_delete=models.CASCADE,
        verbose_name="Structure de Garde",
    )

    class Meta:
        verbose_name = "News"
        verbose_name_plural = "News"

    def __str__(self):
        return self.title


import numpy as np
import tensorflow as tf
import random
import time
from sklearn.metrics import confusion_matrix

from models.CNNModels import Net
from utils import log, extract_minibatch
from config.config import DEFAULT_LOG_ENV, NB_OF_EPOCHS_FOR_BAYESIAN

def train_model(X, Y, valX, valY,
                net: Net,
                model_ckpt_path,
                batch_size=128,
                epochs=50,
                save_step=10,
                display_step=1,
                restore=False,
                model_ckpt_path_to_restore=None,
                cost_sensitive_loss=False,
                cost_matrix=np.array([[0, 1.1, 1.3], [1.1, 0, 1.1], [1.3, 1.1, 0]]),
                is_bayesian=False,
                **kwargs):
    """
    :param X: samples, encoded images
    :param Y: one-hot-encoded labels
    :param valX: same as above used for validation
    :param valY: same as above used for validation
    :param net: Net object
    :param model_ckpt_path: path to save the checkpoints
    :param batch_size: size of one batch
    :param epochs: number of total epochs to train
    :param save_step: model will be saved every save_step epochs, set to 0 to save only at the end, set to -1 to never save
    :param display_step: display lags every display_step epochs
    :param restore: set to True to restore a previous model
    :param model_ckpt_path_to_restore:  path to use to restore a model checkpoint
    :param cost_sensitive_loss: boolean, will implement cost sensitive cross entropy loss function if true.
     See Kukar and Kononenko (1998) for details.
    :param cost_matrix: a matrix of size output_dimension by output_dimension, default to be identity. (i,j)-th element is a
                         penalty factor assigned for misclassifying i to j. It used to combine sample prior
                         (distribution of classes in the training sample) to compute expected penalty of misclassifying i-th class.
                         See Kukar and Kononenko (1998) for details
    :param is_bayesian: boolean, if true, will ensemble the result from the last NB_OF_EPOCHS_FOR_BAYESIAN epochs of training
    :return:
            1. prediction of validation data based on current model
            2. Training loss of current model
            3. validation loss of current model
    """

    # Get sample distribution to calculate expected penalty
    sample_size, _, image_size, channel_size = X.shape

    # number of batches
    nb_of_batches_training = max(int(sample_size / batch_size), 1)

    # calculate expected penalty if we consider a cost sensitive  loss
    if cost_sensitive_loss:
        sample_prior = Y.sum(axis=0) / len(Y)
        expected_penalty = np.dot(cost_matrix, sample_prior) / (1 - sample_prior)
    else:
        expected_penalty = None
    kwargs.update({'expected_penalty': expected_penalty})

    if restore:
        tf.reset_default_graph()
        sess = tf.Session()
        graph = tf.get_default_graph()
        net.restore_importants_ops(sess, model_ckpt_path_to_restore)
        return _run_epochs(restore, sess, net, X, Y, valX, valY, sample_size, epochs, nb_of_batches_training,
                           batch_size, display_step, save_step, model_ckpt_path, is_bayesian)

    else:
        with tf.Graph().as_default():
            with tf.variable_scope(net.name, reuse=None):
                net.build_operations(**kwargs)
                with tf.Session() as sess:
                    sess.run(net.init)
                    return _run_epochs(restore, sess, net, X, Y, valX, valY, sample_size, epochs,
                                       nb_of_batches_training, batch_size, display_step, save_step, model_ckpt_path,
                                       is_bayesian)

def _run_epochs(restore, sess, net, X, Y, valX, valY, sample_size, epochs,
                nb_of_batches_training, batch_size, display_step,
                save_step, model_ckpt_path, is_bayesian):
    """
    Runs the training, See train_model docstring for details
    """
    writer = tf.summary.FileWriter('logs', sess.graph)

    dropout = net.get_dropout()

    training_loss, val_loss, pred = [], [], []
    for e in range(epochs):
        # Display the epoch beginning
        if (e + 1) % display_step == 0:
            text_to_print = '\n***************** Epoch: {:03d}/{:03d} *****************'.format(e + 1,
                                                                                epochs)
            log(text_to_print, DEFAULT_LOG_ENV)

        epoch_start_time = time.time()

        # Shuffle dataset before doing an epoch of training # todo see if we delete it
        Z = list(zip(X, Y))
        random.shuffle(Z)
        X, Y = zip(*Z)

        cc, aa = 0, 0
        for b in range(nb_of_batches_training):
            x_b, y_b = extract_minibatch(X, Y, batch_size=batch_size, current_batch=b)
            _, __, c, a = sess.run([net.optimizer, net.global_step, net.loss, net.accuracy],
                                   feed_dict={net.x: x_b, net.y: y_b, net.phase_train: True, net.dropout: dropout})
            # Make evaluation on a batch basis
            cc += (c * len(y_b))  # len(y_b) since the last batch can be of size smaller than batch size
            aa += (a * len(y_b))
            if (b + 1) % max(int(0.1 * nb_of_batches_training), 1) == 0:
                log('Batch {}/{} done ***  CumLoss'
                    ': {:.4f} CumAccuracy: {:.4f}'.format(b + 1, nb_of_batches_training,
                                                          cc / (batch_size * (b + 1)),
                                                          aa / (batch_size * (b + 1))), DEFAULT_LOG_ENV)

        epoch_loss, epoch_acc = cc / float(sample_size), aa / float(sample_size)
        training_loss.append(epoch_loss)

        # run validation in batches to avoid memory problems
        size_1_image = np.prod(valX[0].shape)
        limit_size = (10 * batch_size) * 42 * 42 * 5
        size_batch = int(limit_size / size_1_image) + 1
        output_val, epoch_val_loss = np.zeros((0, 3)), 0.
        nb_batches = round(len(X) / size_batch)
        log('To avoid killing the kernel, we run predictions in {} batches'.format(nb_batches),
            environment=DEFAULT_LOG_ENV)
        for batch in range(0, len(X), size_batch):
            X_batch, Y_batch = valX[batch:batch + size_batch], valY[batch:batch + size_batch]
            if X_batch.shape[0] == 0:
                break
            pred_batch, epoch_val_loss_batch = sess.run([net.output, net.loss],
                                                        feed_dict={net.x: X_batch, net.y: Y_batch,
                                                                   net.phase_train: False, net.dropout: 0.})
            output_val = np.concatenate([output_val, pred_batch])
            epoch_val_loss += epoch_val_loss_batch
        epoch_val_loss /= nb_batches

        conf_matrix = get_confusion_matrix(valY, output_val)
        rows_conf_matrix = ['{}\n'.format(row_i) for row_i in conf_matrix]
        log('Confusion Matrix : \n ' + ''.join(rows_conf_matrix), environment=DEFAULT_LOG_ENV)
        pred.append(output_val)
        val_loss.append(epoch_val_loss)
        epoch_val_acc = np.trace(conf_matrix) / np.sum(conf_matrix)

        # Displaying training info for the epoch
        if (e + 1) % display_step == 0:
            text_to_print = "Training time: {}  ======== Loss: {:.4f} Accuracy: {:.4f} val_acc: {:.4f} ".format(
                round(time.time() - epoch_start_time, 2), epoch_loss, epoch_acc, epoch_val_acc)
            log(text_to_print, DEFAULT_LOG_ENV)

        # Saving the model
        if save_step >= 1 and (e + 1) % save_step == 0:
            net.save(sess, model_ckpt_path, verbose=True, epoch=e + 1, write_meta_graph=(not restore))

        # Adding values to summary
        summary_str = sess.run(net.summary_op,
                               feed_dict={net.x: x_b, net.y: y_b, net.phase_train: True, net.dropout: dropout})
        writer.add_summary(summary_str, sess.run(net.global_step))
    if save_step >= 0:
        net.save(sess, model_ckpt_path, verbose=True, epoch=epochs, write_meta_graph=True)

    writer.close()

    if is_bayesian:
        # Sample from the last NB_OF_EPOCHS_FOR_BAYESIAN epochs to form Bayesian learning:
        return (np.mean(pred[-NB_OF_EPOCHS_FOR_BAYESIAN:], axis=0), training_loss, val_loss)
    else:
        return (pred[-1], training_loss, val_loss)

def get_confusion_matrix(ytrue, ypred):
    """
    Create confusion matrix based on true results ytrue and predicted result ypred
    """
    ytrue = np.argmax(ytrue, axis=1)
    ypred = np.argmax(ypred, axis=1)
    return confusion_matrix(ytrue, ypred)


#!/usr/bin/python3
import glob, hashlib, json, yaml

bishop_working_dict = {"all": []}
wordlist_working_dict = {"all": []}

files = glob.glob("./sources/**/*.yml", recursive=True)

def bishop_append(title, entry, source_class):
    global bishop_working_dict

    # allow for unvalidate-able sources to be added to wordlists, but not bishop
    if "validation" not in entry.keys() or "path" not in entry.keys():
        return

    bishop_working_entry = {
        "description": entry["description"],
        "enabled": True,
        "name": title,
        "risk": "low",
        "searchString": entry["validation"],
        "url": entry["path"],
    }

    m = hashlib.md5()
    m.update(json.dumps(bishop_working_entry).encode("utf-8"))
    bishop_working_entry["uid"] = m.hexdigest()

    if source_class not in bishop_working_dict.keys():
        bishop_class_tmp = []
    else:
        bishop_class_tmp = bishop_working_dict[source_class]

    bishop_class_tmp.append(bishop_working_entry)
    bishop_working_dict[source_class] = bishop_class_tmp

def wordlist_append(title, entry, source_class):
    global wordlist_working_dict

    if "path" not in entry.keys():
        return

    if source_class not in wordlist_working_dict.keys():
        wordlist_class_tmp = []
    else:
        wordlist_class_tmp = wordlist_working_dict[source_class]

    wordlist_class_tmp.append(entry["path"])
    wordlist_working_dict[source_class] = wordlist_class_tmp

def add_to_multi_dict(title, entry, source_class):
    bishop_append(title, entry, "all")
    bishop_append(title, entry, source_class)

    wordlist_append(title, entry, "all")
    wordlist_append(title, entry, source_class)

for file_path in files:
    file_components = file_path.split("/")
    source_class = file_components[-2]

    with open(file_path, "r") as stream:
        data_loaded = yaml.safe_load(stream)

        for recon_title, recon_dict in data_loaded.items():
            add_to_multi_dict(recon_title, recon_dict, source_class)

for bishop_class, bishop_data in bishop_working_dict.items():
    out_file = f"./bishop/{bishop_class}.json"
    with open(out_file, "w") as json_target:
        json.dump(bishop_data, json_target, indent=4)

for wordlist_class, wordlist_data in wordlist_working_dict.items():
    out_file = f"./wordlists/{wordlist_class}.txt"
    wordlist = ""
    for word in wordlist_data:
        wordlist = f"{wordlist}{word}\n"
    wordlist.strip("\n")
    with open(out_file, "w") as wordlist_target:
        wordlist_target.write(wordlist)


import json
from typing import Sequence

from telegram import Update, InlineKeyboardButton, InlineKeyboardMarkup, ReplyKeyboardRemove, ReplyKeyboardMarkup
from telegram.ext import ContextTypes, CallbackContext, ConversationHandler, CommandHandler, CallbackQueryHandler, MessageHandler, filters

from Constants.NumericConstants import MAX_FAILS

from Controllers.UserController import UserController
from Controllers.ReservationController.ConversationStates import ConversationStates

from Models.Reservation import Reservation
from Models.Person import Person
from Helpers.Helper import Helper

# Reservation management controller
class ReservationController:

    @staticmethod
    async def select_person_for_reservation(update: Update, context: ContextTypes.DEFAULT_TYPE):
        telegram_id = update.effective_user.id
        persons = Person.get_all_persons_by_telegram_id( telegram_id )
        persons_reply_markup: Sequence[Sequence[str]] = [[]]
        answer = "*✒ A que persona vas a reservar?:* \n------------------------------------------\n"

        if( not persons ):
            await update.message.reply_text(":( Usted no ha registrado a ninguna persona todavia")
            return

        persons_reply_markup, answer = Helper.get_persons_keyboard_from_user(update, context)

        await update.message.reply_text(answer, parse_mode="Markdown", reply_markup=ReplyKeyboardMarkup(persons_reply_markup))
        return ConversationStates.CREATE_RESERVATION

    @staticmethod
    async def get_reservations(update: Update, context: ContextTypes.DEFAULT_TYPE):
        telegram_id = update.effective_user.id
        reservations = Reservation.get_all_reservations_by_telegram_id(telegram_id)

        if(not reservations):
            await update.message.reply_text("🚧 *No tienes ninguna reserva*", parse_mode="Markdown")
            return

        for reservation in reservations:
            ( reservation_id, arrival_order, name) = reservation
            action = { "name": "CANCEL_RESERVATION", "pk": reservation_id }
            cancel_reservation_button = InlineKeyboardButton("🚫🚌 Cancelar", callback_data=json.dumps(action))
            failNumber = "" if arrival_order <= MAX_FAILS else f"⚠ *Fallo:* {arrival_order - MAX_FAILS}"
            await update.message.reply_text(
                f"🪪 {name}\n*🔢 Turno:* {arrival_order}\n",
                parse_mode="Markdown",
                reply_markup=InlineKeyboardMarkup([[cancel_reservation_button]])
            )

        if( update.message.text == "/reservar_todos" ):
            await update.message.reply_text("✅ Todas las personas han sido reservadas con exito!")

    @staticmethod
    async def create_reservation_for_all_person(update: Update, context: ContextTypes.DEFAULT_TYPE):
        telegram_id = update.effective_user.id
        persons = Person.get_all_persons_by_telegram_id( telegram_id )
        await UserController.register_user(update, context)
        for person in persons:
            (id, *_) = person
            if( not Reservation.exist_person(id) ):
                Reservation.create_reservation( telegram_id, id )
        await ReservationController.get_reservations(update, context)

    # Create a unique reservation by User
    @staticmethod
    async def create_reservation(update: Update, context: ContextTypes.DEFAULT_TYPE) -> None:

        telegram_id = update.message.from_user.id
        person_name = update.message.text

        person = Person.get_person_id_by_name_from_telegram_id(telegram_id, person_name)

        if(not person):
            await update.message.reply_text(f"🫤 Lo siento, no hay personas registradas con el nombre {person_name}", reply_markup=ReplyKeyboardRemove())
            return ConversationHandler.END

        person_id, = person
        await UserController.register_user(update, context)
        if( not Reservation.exist_person(person_id) ):
            Reservation.create_reservation(telegram_id, person_id)
            await update.message.reply_text(f"✅ Se ha creado una reserva satisfactoriamente para: {person_name}", reply_markup=ReplyKeyboardRemove())
        else:
            await update.message.reply_text("😉 Ya esa persona estaba reservada: ", reply_markup=ReplyKeyboardRemove())
        reservation_id, arrival_order, name = Reservation.get_reservation_by_user_id_and_name(telegram_id, person_name)

        action = { "name": "CANCEL_RESERVATION", "pk": reservation_id }
        cancel_reservation_button = InlineKeyboardButton("🚫🚌 Cancelar", callback_data=json.dumps(action))
        failNumber = "" if arrival_order <= MAX_FAILS else f"⚠ *Fallo:* {arrival_order - MAX_FAILS}"

        await update.message.reply_text(
            f"🪪 {person_name}\n*🔢 Turno:* {arrival_order}\n{failNumber}",
            parse_mode="Markdown",
            reply_markup=InlineKeyboardMarkup([[cancel_reservation_button]])
        )
        return ConversationHandler.END

    @staticmethod
    async def cancel_reservation(update: Update, context: CallbackContext):
        await update.effective_message.delete()
        action = update.callback_query.data

        if( not Helper.is_json( action ) ):
            return

        name, pk = json.loads(action).values()

        if( name != "CANCEL_RESERVATION" ):
            return

        reservation = Reservation.get_reservation_by_id(pk)

        if(not reservation):
            await update.effective_message.reply_text(f"⛔🕘 Esa reservación ya no existe, vuelva a reservar")
            return

        reservation_id, arrival_order, name = Reservation.get_reservation_by_id(pk)
        Reservation.delete_by_id(reservation_id)
        await update.effective_message.reply_text(f"✅ La reservacion de {name} se ha cancelado con exito ! \n 🗑 Eliminado el turno: {arrival_order}")

    # CANCEL_OPERATION
    @staticmethod
    async def cancel_operations(update: Update, context: ContextTypes.DEFAULT_TYPE) -> int:
        await update.message.reply_text( "Operacion Cancelada", reply_markup=ReplyKeyboardRemove() )
        return ConversationHandler.END

reservation_conversation_handler = ConversationHandler(
    entry_points=[
        CommandHandler("reservas", ReservationController.get_reservations),
        CommandHandler("reservar_todos", ReservationController.create_reservation_for_all_person),
        CommandHandler("reservar", ReservationController.select_person_for_reservation)
    ],
    states={
        ConversationStates.CREATE_RESERVATION: [MessageHandler(filters.TEXT & ~filters.COMMAND, ReservationController.create_reservation)]
    },
    fallbacks=[
        CommandHandler("cancel", ReservationController.cancel_operations),
        MessageHandler(filters.ALL, ReservationController.cancel_operations)
    ]
)

reservation_callback_query = CallbackQueryHandler(ReservationController.cancel_reservation)


# vizualizace vzorkovaci frekvence
import numpy as np
import matplotlib.pyplot as plt

# vytvoreni datovych vektoru
x = np.linspace(0, 10 * np.pi, 1000)
y = np.sin(x)

# pole nasobku pi pro upraveni f_vz
consts = [2.1, 1, 0.2]

# smycka pro vypocet a vykresleni 3 datovych rad
for i in range(3):
    f_x = []
    f_y = []
    idx = 1
    f = 1

    # vypocet dilcich vektoru pro vzorkovani
    while f < np.max(x):
        f_x.append(f)
        f_y.append(np.sin(f))
        f += consts[i] * np.pi
        idx += 1

    # vykresleni grafu
    plt.subplot(3, 1, i + 1)
    plt.plot(x, y)
    plt.scatter(f_x, f_y, color='red')
    plt.plot(f_x, f_y)
    comment = ''
    if consts[i] == 1:
        comment = ' ... Nyquistova frekvence'
    plt.title(f'f_max = {consts[i] / 2} * f_vz{comment}', fontsize=15)
    plt.grid()

    if i == 2:
        plt.xlabel('Čas [s]')

    plt.ylabel('Amplituda [-]')

plt.subplots_adjust(hspace=0.37)
plt.show()


from flask import Flask,redirect,url_for,render_template,request
import numpy as np
import pickle
import sklearn
import os

file_path = 'amit.pkl'

if os.path.isfile(file_path):
    print("File exists")
else:
    print("File does not exist")

model = pickle.load(open('amit.pkl', 'rb'))

app = Flask(__name__)

@app.route('/')
def welcome():
    return render_template('index.html')

@app.route('/predict',methods = ["POST"])
def home():
    sy1 = request.form['s1']
    sy2 = request.form['s2']
    sy3 = request.form['s3']
    sy4 = request.form['s4']
    sy5 = request.form['s5']
    sy6 = request.form['s6']
    sy7 = request.form['s7']
    sy8 = request.form['s8']
    sy9 = request.form['s9']
    sy10 = request.form['s10']

    arr = np.array([[sy1,sy2,sy3,sy4,sy5,sy6,sy7, sy8, sy9,sy10]])
    pred = model.predict(arr)
    return render_template('result.html',data=pred)
if __name__ == '__main__':
    app.run(debug=True)


try:
    import os
    import json
    import glob
    import argparse
    import random

    import numpy as np
    from scipy import signal as sg
    from scipy.ndimage.filters import maximum_filter
    import matplotlib.pyplot as plt
    import matplotlib.image as mpimg
    import scipy.misc
    from PIL import Image

    import matplotlib.pyplot as plt
except ImportError:
    print("Need to fix the installation")
    raise

K = 3

def pick_k_coordinates(lst, k):
    return None if not len(lst) else tuple(lst[i] for i in range(0, len(lst), len(lst) // k))

def pick_k_random_coordinates(lst, k):
    return None if not len(lst) else tuple(lst[i] for i in random.sample(range(0, len(lst)), k))

def find_pixels(img, real_image):
    lights = np.where(img == 19)
    lights = list(zip(lights[0], lights[1]))
    non_lights = np.where(img != 19)
    non_lights = list(zip(non_lights[0], non_lights[1]))
    coordinates_lights = pick_k_coordinates(lights, K)
    if coordinates_lights:
        cut_image(real_image, [coordinates_lights, pick_k_random_coordinates(non_lights, K)])
    # return [coordinates_lights, pick_k_random_coordinates(non_lights, K)] if coordinates_lights else []

def get_files_from_base(path, file_suffix):
    print(path, file_suffix)
    s = glob.glob(os.path.join(path, file_suffix))
    print(s)
    return s

def write_to_bin_file(value, filename):
    with open(f"./data_bin/{filename}.bin", 'wb') as file:
        file.write((value).to_bytes(24, byteorder='big', signed=False))

def write_image_to_bin_file(image, filename):
    image.astype('uint8').tofile(f"./data_bin/{filename}.bin")

def cut_image(img, pixels):
    bord_img = add_border(img)
    for coordinate in pixels[0]:
        x = coordinate[0]
        y = coordinate[1]
        write_to_bin_file(1, 'aachen_val')
        write_image_to_bin_file(bord_img[x - 40:x + 40, y - 40:y + 40, :], 'aachen_img')

    for coordinate in pixels[1]:
        x = coordinate[0]
        y = coordinate[1]
        write_to_bin_file(0, 'aachen_val')
        write_image_to_bin_file(bord_img[x - 40:x + 40, y - 40:y + 40, :], 'aachen_img')

def add_border(image):
    length, width, layers = image.shape
    length_array, width_array = np.zeros((length + 80) * 40 * 3).reshape((length + 80, 40, 3)), np.zeros(
        40 * width * 3).reshape((40, width, 3))
    image_full = np.vstack([image, width_array])
    image_full = np.vstack([width_array, image_full])
    image_full = np.hstack([length_array, image_full])
    image_full = np.hstack([image_full, length_array])
    return image_full

def main(argv=None):
    label_bases = [('./gtFine/train/aachen', '*_labelIds.png'), ('./gtFine/val', '*_labelIds.png')]
    img_bases = [('./leftImg8bit/train/aachen', '*_leftImg8bit.png'), ('./leftImg8bit/val', '*_leftImg8bit.png')]
    for i, path in enumerate(label_bases[0:1]):
        flist_1 = get_files_from_base(path[0], path[1])

    for i, path in enumerate(img_bases[0:1]):
        flist_2 = get_files_from_base(path[0], path[1])

    for index, image in enumerate(flist_1[:2]):
        print("f", flist_1)
        img_1 = np.array(Image.open(image)).astype('uint8')
        img_2 = np.array(Image.open(flist_2[index])).astype('uint8')
        find_pixels(img_1, img_2)
        plt.figure().clf()
        plt.imshow(img_1)
        plt.show(block=True)

def read_files(root_dir):

    files = glob.glob(root_dir + '/**/*.json', recursive=True)
    return files

if __name__ == '__main__':
    read_files('gtFine')