import pandas as pd
import numpy as np
from matplotlib import pyplot as plt
from matplotlib.dates import (DAILY, DateFormatter, rrulewrapper, RRuleLocator, drange, num2date)
import datetime
from datetime import timedelta
from scipy.optimize import curve_fit
import os

'''
Supporting Functions
'''

def sigmoid(x, L ,x0, k, b):
    y = L / (1 + np.exp(-k*(x-x0)))+b
    return (y)

def quadratic (x, a, b, h, c):
    return(a*((x-h)**2) + b*(x-h) + c)

def exponential(x, a, b, h, c):
    return(a*(b ** (x-h)) + c)

def fit(y, X, X_pred, p0_sig=None, p0_quad=None, p0_exp=None, tr_solver=None, method='trf'):
    FTOL = [1e-5, 1e-4, 1e-3, 1e-2, 1e-1]
    ps = np.array([0,0,0,0])
    pq = np.array([0,0,0,0])
    pe = np.array([0,1,0,0])

    for tol in FTOL:
        try:
            ps, _ = curve_fit(sigmoid, X, y, p0=p0_sig, ftol=tol, method=method)
            break
        except:
            continue

    for tol in FTOL:
        try:
            pq, _ = curve_fit(quadratic, X, y, p0=p0_quad, ftol=tol, method=method, bounds=([0, -np.inf, -np.inf, -np.inf], np.inf))
            break
        except:
            continue

    for tol in FTOL:
        try:
            pe, _ = curve_fit(exponential, X, y, p0=p0_exp, ftol=tol, method=method)
            break
        except:
            continue

    sig = sigmoid(X_pred, *ps)
    quad = quadratic(X_pred, *pq)
    expo = exponential(X_pred, *pe)

    return sig, quad, expo, ps, pq, pe

def predictCases(data, days_predicted = 35, vis=True):
    ##  Input
    #   data: confirmed cases from usafacts
    #   days_predicted: number of days to predict
    #   vis: view graphs

    ##  Output
    #   X: number of days since first case
    #   X_pred: number of days since first case including predicted days
    #   y: number of cases
    #   y_pred: number of cases predicted
    #   dates: actual dates
    #   dates_pred: actual dates including predicted daysill

    # Parameters
    learn_rate = 0.125
    w1=0.5
    w2=0.5
    z1 = w1
    z2 = w2
    learn_period = 14

    county = data['County Name'] + ', ' + data['State'] # County name
    y = np.array(data.to_list()[4:]).astype(np.float) #Number of cases
    y_orig = y
    # Create dates
    start_date = datetime.date(2020,1,22)
    end_date = start_date + timedelta(days=len(y))
    delta = datetime.timedelta(days=1)
    dates = drange(start_date, end_date, delta)
    end_date_pred = start_date + timedelta(days=len(y)+days_predicted)
    dates_pred = drange(start_date, end_date_pred, delta)
    dates_str = [item.strftime('%m/%d/%Y') for item in num2date(dates_pred) ]

    # Need to remove leading 0's and make sure there is enough data to find a meaningful curve fit
    first_case_idx = np.nonzero(y)[0][0] # index of first case
    y_first = y[first_case_idx:] #data since first case
    # Learn from N days
    if len(y_first) < learn_period:
        learn_period = 3
    y_idx_start = len(y_first)-learn_period

    # Initialize variables to sum cumulative observed error for comparing exponential vs quadratic
    quad_pred_err_tot = 0
    exp_pred_err_tot = 0

    # Loop through dates to fit and learn from
    for idx, y_idx in enumerate(np.arange(y_idx_start,len(y_first)+1)):
        y = y_first[:y_idx]
        X = np.linspace(0,len(y)-1,len(y)) #Days
        X_pred = np.linspace(0,len(y)-1+days_predicted,len(y)+days_predicted) #Days
        # Create dates

        if (idx == 0):
            p0_sig = [max(y_first), np.median(X),1,min(y_first)]
            p0_quad = None
            p0_exp = None
        else:
            p0_sig = ps
            p0_quad = pq
            p0_exp = pe

        sig, quad, exp, ps, pq, pe = fit(y, X, X_pred, p0_sig, p0_quad, p0_exp)
        y_pred_quad = w1*quad + w2*sig
        y_pred_exp = z1*exp + z2*sig

        if (y_idx < len(y_first)):
            sig_err = np.sum((y_first[y_idx:] - sig[y_idx:len(y_first)]) ** 2)
            quad_err = np.sum((y_first[y_idx:] - quad[y_idx:len(y_first)]) ** 2)
            exp_err = np.sum((y_first[y_idx:] - exp[y_idx:len(y_first)]) ** 2)

            if (idx > 2):
                quad_pred_err_tot += np.sum((y_first[y_idx:] - y_pred_quad[y_idx:len(y_first)]) ** 2)
                exp_pred_err_tot += np.sum((y_first[y_idx:] - y_pred_exp[y_idx:len(y_first)]) ** 2)

            tot_error = sig_err + quad_err
            w1_new = 1 - (quad_err/tot_error)
            w2_new = 1 - (sig_err/tot_error)

            tot_error = sig_err + exp_err
            z1_new = 1 - (exp_err/tot_error)
            z2_new = 1 - (sig_err/tot_error)

            # Save for plotting
            w1_old = np.copy(w1)
            w2_old = np.copy(w2)
            z1_old = z1
            z2_old = z2

            # Update weights
            w1 = w1 + (w1_new - w1) * learn_rate
            w2 = w2 + (w2_new - w2) * learn_rate
            z1 = z1 + (z1_new - z1) * learn_rate
            z2 = z2 + (z2_new - z2) * learn_rate

        if (vis):
            dates_pred_trunc = dates_pred[first_case_idx:first_case_idx+len(X_pred)]
            # Plot
            fig = plt.figure(figsize=(8, 6))
            ax = fig.add_subplot(1, 1, 1)
            ax.plot_date(dates[first_case_idx:first_case_idx+y_idx], y, color='#e74c3c', ls='solid', marker='o')
            try:
                ax.plot_date(dates[first_case_idx+y_idx:first_case_idx+len(y_first)], y_first[y_idx:], color='#636e72', ls='solid', marker='o')
            except:
                pass
            ax.plot_date(dates_pred_trunc, y_pred_quad, color='#00b894', ls='solid',marker="")
            ax.plot_date(dates_pred_trunc, y_pred_exp, color='#2d3436', ls='solid',marker="")
            ax.plot_date(dates_pred_trunc, sig, color='#0984e3', ls='dashed', marker="")
            ax.plot_date(dates_pred_trunc, quad, color='#6c5ce7', ls='dashed', marker="")
            ax.plot_date(dates_pred_trunc, exp, color='#d63031', ls='dashed', marker="")

            w1r = np.round(w1_old, decimals=2)
            w2r = np.round(w2_old, decimals=2)
            z1r = np.round(z1_old, decimals=2)
            z2r = np.round(z2_old, decimals=2)

            # Format plot
            formatter = DateFormatter('%m/%d')
            rule = rrulewrapper(DAILY, interval=7)
            loc = RRuleLocator(rule)
            ax.xaxis.set_major_locator(loc)
            ax.set_xlabel('Days')
            ax.set_ylabel('Confirmed Cases')
            ax.set_title(county + '-- Q: ' + str(w1r) + ' L1: ' + str(w2r) + ' E: ' + str(z1r) + ' L2: ' + str(z2r))
            ax.xaxis.set_major_formatter(formatter)
            ax.xaxis.set_tick_params(rotation=30, labelsize=8)
            ax.legend(['True [Training]', 'True [Testing]', 'Predicted-Quad.', 'Predicted-Exp.','Logistic', 'Quadratic', 'Exponential'])
            ax.set_ylim([0, np.max(y_first) * 1.5])

    if (quad_pred_err_tot < exp_pred_err_tot):
        y_pred = y_pred_quad
        # growth = 'quadratic'
    else:
        y_pred = y_pred_exp
        # growth = 'exponential'

    pad_width = len(y_orig)- len(y)
    y = np.pad(y, (pad_width,0), 'constant')
    y_pred = np.pad(y_pred, (pad_width,0), 'constant')
    y_pred[y_pred<0] = 0
    pad_width = len(y_pred)- len(y)
    y = np.pad(y, (0,pad_width), 'constant', constant_values=(-1,-1))

    X = np.linspace(0,len(y)-1,len(y)) #Days
    X_pred = np.linspace(0,len(y)-1+days_predicted,len(y)+days_predicted) #Days
    # Create dictionary output
    case_output = {
            # 'X': X,
            # 'X_pred': X_pred,
            'y': y,
            'y_pred': y_pred,
            # 'dates': dates,
            # 'dates_pred':dates_pred,
            'dates_str': dates_str,
            }
    return(case_output)

def check_sufficient_data(county_data):
    enough_data = True
    y = np.array(county_data.to_list()[4:]).astype(np.float) #Number of cases
    ydif = np.diff(y, prepend=[0])
    if(len(np.nonzero(ydif)[0]) < 10):
        enough_data = False
    return(enough_data)

def read_data(file_path='./data/covid_confirmed_usafacts.csv', county_file_path='./data/county_data.csv'):
    data = pd.read_csv(file_path,delimiter=',',converters={'countyFIPS': str})

    data = data.sort_values(by=data.columns[-1], ascending=False)
    data = data.reset_index(drop=True)
    data['countyFIPS'] = data['countyFIPS'].apply(lambda x: x.zfill(5))
    county_data = pd.read_csv(county_file_path,delimiter=',')
    county_data['GEO_ID'] = county_data['GEO_ID'].str.strip().str[-5:]

    return(data, county_data)

##############################################################################
data, county_data = read_data(file_path='./data/usafacts_cases.txt')
data = data[data['countyFIPS'] != '00000']
data = data[data['countyFIPS'] != '00001']

output=[]
vis = False

# import time
# start = time.time()

data_fips = data['countyFIPS'].tolist()
# data_fips = ['36081', '36061', '48023']
# data_fips = ['05081']
# data_fips = ['17031']

for idx, code in enumerate(data_fips):
    print(idx)
    row = data[data['countyFIPS']==code].iloc[0]
    if (check_sufficient_data(row)):
        result = predictCases(row, days_predicted = 28, vis=vis)
        df = pd.DataFrame(result)
        df['countyFIPS'] = row['countyFIPS']
        df['County Name'] = row['County Name']
        df['State'] = row['State']
        df['stateFIPS'] = row['stateFIPS']
    else:
        dates_str = result['dates_str']
        y_pred = np.zeros(len(dates_str))
        y = np.array(row.to_list()[4:]).astype(np.float) #Number of cases
        pad_width = len(y_pred)- len(y)
        y = np.pad(y, (0,pad_width), 'constant', constant_values=(-1,-1))
        df = pd.DataFrame({'y': y, 'y_pred': y_pred, 'dates_str': dates_str})
        df['countyFIPS'] = row['countyFIPS']
        df['County Name'] = row['County Name']
        df['State'] = row['State']
        df['stateFIPS'] = row['stateFIPS']
    if (idx==0):
        df_agg = df
    else:
        df_agg = pd.concat([df_agg,df])

data_fips = pd.unique(data['State']).tolist()

for idx, code in enumerate(data_fips):
    print(idx)
    temp = data[data['State']==code]
    row = temp.sum()
    row['countyFIPS'] = temp['stateFIPS'].iloc[0]
    row['County Name'] = temp['State'].iloc[0]
    row['State'] = temp['State'].iloc[0]
    row['stateFIPS'] = temp['stateFIPS'].iloc[0]

    if (check_sufficient_data(row)):
        result = predictCases(row, days_predicted = 28, vis=vis)
        df = pd.DataFrame(result)
        df['countyFIPS'] = row['countyFIPS']
        df['County Name'] = row['County Name']
        df['State'] = row['State']
        df['stateFIPS'] = row['stateFIPS']

    df_agg = pd.concat([df_agg,df])

# end = time.time()
# total = end-start
df_agg.to_csv('./data/predicted_cases.csv',index=False)


import functools
import pathlib
from collections import OrderedDict
from collections import namedtuple

import yaml

@functools.lru_cache(maxsize=1)
def get_publishings():
    from publish_python.cli import artifact_handlers
    from publish_python.cli import upload_handlers

    config_path = pathlib.Path('publish-python.yaml')
    if not config_path.exists():
        raise RuntimeError(
            'Must be invoked in a directory containing a '
            f"'{config_path}' file")
    data = yaml.safe_load(config_path.read_bytes())
    assert 'artifacts' in data
    assert isinstance(data['artifacts'], list)

    Artifact = namedtuple('Publishing', ['type', 'config', 'uploads'])
    Upload = namedtuple('Upload', ['type', 'config'])

    artifacts = OrderedDict()
    for a in data['artifacts']:
        assert isinstance(a, dict), type(a)
        assert 'type' in a, str(a)
        assert isinstance(a['type'], str), type(a['type'])
        assert a['type'] in artifact_handlers, str(a['type'])
        assert isinstance(a.get('config', {}), dict), type(a['config'])
        assert a['type'] not in artifacts
        artifact = Artifact(
            type=a['type'],
            config=a.get('config', {}),
            uploads=OrderedDict())

        assert 'uploads' in a
        assert isinstance(a['uploads'], list), type(a['uploads'])
        for u in a['uploads']:
            assert 'type' in u
            assert isinstance(u['type'], str), type(u['type'])
            assert u['type'] in upload_handlers, str(u['type'])
            assert isinstance(u.get('config', {}), dict), type(u['config'])

            upload = Upload(type=u['type'], config=u.get('config', {}))

            artifact.uploads[upload.type] = upload
        artifacts[artifact.type] = artifact

    return artifacts


"""
Write a function that takes directory path, a file extension and an optional tokenizer.
It will count lines in all files with that extension if there are no tokenizer.
If a the tokenizer is not none, it will count tokens.

For dir with two files from hw1.py:
# >>> universal_file_counter(test_dir, "txt")
6
# >>> universal_file_counter(test_dir, "txt", str.split)
6

"""
import os
from pathlib import Path
from typing import Callable, Optional

DIR_PATH = Path(__file__).parent

def count_len(tokens):
    try:
        if isinstance(tokens, str):
            return 1
        return len(tokens)
    except TypeError:
        return 1

def universal_file_counter(dir_path: Path, file_extension: str, tokenizer: Optional[Callable] = None) -> int:
    counter = 0

    for file in os.listdir(dir_path):
        if file.endswith(file_extension):
            with open(dir_path / file) as file_read:
                if tokenizer is None:
                    counter += len(file_read.readlines())
                else:
                    counter += sum(count_len(tokenizer(line)) for line in file_read.readlines())

    return counter


import requests
import json
import os

access_key = os.getenv('ACCESS_KEY')
base_url = 'https://api.unsplash.com/'

def download_image(url, file_path, output):
    response = requests.get(url)
    with open(output + '/' + file_path, 'wb') as file:
        file.write(response.content)

def search_and_download(query, count, output):
    search_url = base_url + 'search/photos'
    headers = {'Authorization': 'Client-ID ' + access_key}
    params = {'query': query, 'per_page': count}
    response = requests.get(search_url, headers=headers, params=params)
    data = json.loads(response.text)
    # print(data)

    for i, photo in enumerate(data['results']):
        image_url = photo['urls']['regular']
        file_path = f'image_{i}.jpg'
        download_image(image_url, file_path, output)

def main():
    query = input("Enter your search query: ")
    count = int(input("Enter the number of images to download: "))

    cur_dir = os.getcwd()
    output = cur_dir + f'/{query}'
    if not os.path.exists(output):
        os.mkdir(output)

    search_and_download(query, count, output)

if __name__ == '__main__':
    main()


from django.conf import settings
from django.http import HttpResponse, HttpRequest, Http404
from user_profile.models import User
from django.shortcuts import get_object_or_404
import jwt
import time
from django.core.exceptions import PermissionDenied

def generate_jwt(lifespan: int, id: int) -> str:
    '''Function that will generate a jwt token with the id and secret'''
    current_time: int = int(time.time())
    payload: dict[str, any] = {
        "iss": "pong99",
        "sub": id,
        "exp": current_time + lifespan,
        "iat": current_time
    }
    return jwt.encode(payload, settings.SECRET_KEY, algorithm="HS256")

def first_token(response: HttpResponse, id: int) -> HttpResponse:
    '''Adds an element to the header to signal to the frontend that it is the first token,
       preventing the frontend to fetch twice.
    '''
    response["new"] = "True"
    return add_double_jwt(response, id)

def add_double_jwt(response: HttpResponse, id: int) -> HttpResponse:
    '''Create access and refresh token and add them to the http response'''
    REFRESH_DURATION: int = 7200
    ACCESS_DURATION: int = 30

    access_token = generate_jwt(ACCESS_DURATION, id)
    refresh_token = generate_jwt(REFRESH_DURATION, id)
    response.set_cookie(
        key="refresh_jwt", value=refresh_token, secure=True, httponly=True, samesite="None")
    response["jwt"] = access_token
    return response

def decrypt_user_id(jwt_token: str) -> int:
    '''Decrypt the user id and send it back, if any error happen send
    a negative integer'''
    EXPIRED: int = -1
    INVALID: int = -2

    try:
        decrypt_token: dict = jwt.decode(
            jwt_token, settings.SECRET_KEY, algorithms=["HS256"])
        return decrypt_token.get("sub")
    except jwt.ExpiredSignatureError:
        return EXPIRED
    except jwt.InvalidTokenError:
        return INVALID

def get_user_obj(request: HttpRequest) -> User:
    access_jwt_cookie = request.headers.get("jwt-access")
    if access_jwt_cookie is None:
        raise PermissionDenied("Couldn't locate access jwt")

    decrypt_result = decrypt_user_id(access_jwt_cookie)
    if decrypt_result <= 0:
        # IF 404 DELETE COOKIE ?? IF LOG, MAKE NUKE, MAKE AND TOKEN STILL THERE, WILL NOT FIND USER
        raise Http404("User not found")

    user = get_object_or_404(User, id=decrypt_result)
    return user

def generate_2fa_token(id: int) -> str:
    '''Function that will generate a jwt token with the id and secret'''
    current_time: int = int(time.time())
    lifespan: int = 300
    payload: dict[str, any] = {
        "iss": "pong99",
        "sub": id,
        "exp": current_time + lifespan,
        "iat": current_time
    }
    return jwt.encode(payload, settings.SECRET_KEY, algorithm="HS256")


from typing import Any
from django.core.paginator import Paginator, EmptyPage
from graphql_jwt.decorators import login_required
from django.utils import timezone
from datetime import timedelta
from ..models import Booking, BookingSlot, Movie, MovieFormat
from meta.models import Language
from ..constants import BookingStatus
from .types import PaginatedBookingType, PaginatedMoviesListFormats

class PaginatorBookingResolver:
    @login_required
    def __call__(self, root, info, page: int, limit: int):
        results = Paginator(
            Booking.objects.filter(
                user=info.context.user, status=BookingStatus.BOOKED.value
            )
            .select_related(
                "slot_grp__slot__movie",
                "slot_grp__slot__format",
                "slot_grp__slot__screen__theatre",
            )
            .order_by("-booked_at")
            .all(),
            limit,
        )
        total_pages = results.num_pages
        prev_page = None
        next_page = None
        count = 0
        if page != 1:
            prev_page = page - 1
        if page != total_pages:
            next_page = page + 1
        try:
            results = results.page(page).object_list
            count = len(results)
        except EmptyPage:
            results = []
        return PaginatedBookingType(
            prev_page=prev_page,
            next_page=next_page,
            results=results,
            count=count,
        )

class PaginatedMoviesDetailsResolver:
    @login_required
    def __call__(self, root, info, page: int, limit: int, city):
        movie_langs = list(
            BookingSlot.objects.select_related("screen__theatre__city")
            .filter(
                screen__theatre__city_id=city,
                is_fully_booked=False,
                screening_datetime__gte=timezone.now(),
                screening_datetime__lte=timezone.now() + timedelta(days=7),
            )
            .order_by()
            .values_list("movie", "lang", "format")
            .distinct()
        )
        movie_details_dict = dict()
        for movie_lang in movie_langs:
            movie_id, lang_id, format_id = movie_lang
            if movie_id not in movie_details_dict.keys():
                movie_details_dict[movie_id] = {lang_id: set([format_id])}
            else:
                if lang_id not in movie_details_dict[movie_id].keys():
                    movie_details_dict[movie_id][lang_id] = set([format_id])
                else:
                    movie_details_dict[movie_id][lang_id].add(format_id)

        movies_details_list = []
        for movie_id in movie_details_dict.keys():
            movies_details_list.append({"movie": movie_id, "langs": list()})

        for movie in movies_details_list:
            [
                movie["langs"].append(
                    {
                        "lang": lang_id,
                        "formats": list(
                            movie_details_dict[movie["movie"]][lang_id]
                        ),
                    }
                )
                for lang_id in movie_details_dict[movie["movie"]].keys()
            ]

        for movie_detail in movies_details_list:
            movie_detail["movie"] = Movie.objects.get(pk=movie_detail["movie"])
            for lang_detail in movie_detail["langs"]:
                lang_detail["lang"] = Language.objects.get(
                    pk=lang_detail["lang"]
                )
                lang_detail["formats"] = MovieFormat.objects.filter(
                    id__in=lang_detail["formats"]
                )

        results = Paginator(
            movies_details_list,
            limit,
        )
        total_pages = results.num_pages
        prev_page = None
        next_page = None
        count = 0
        if page != 1:
            prev_page = page - 1
        if page != total_pages:
            next_page = page + 1
        try:
            results = results.page(page).object_list
            count = len(results)
        except EmptyPage:
            results = []

        return PaginatedMoviesListFormats(
            prev_page=prev_page,
            next_page=next_page,
            results=results,
            count=count,
        )


from urllib.request import urlopen
from urllib.error import HTTPError
from urllib.error import URLError
from bs4 import BeautifulSoup

codigo = input("Digite o codigo do aerodromo ou 0 para sair:")
while codigo != "0":
    try:
        url = "https://www.aisweb.aer.mil.br/?i=aerodromos&codigo="+codigo
        page = urlopen(url)
        page_content = BeautifulSoup(page.read(), 'html.parser')

        # cartas
        cartas = []
        next_s = page_content.select_one('h4:contains("Cartas")').find_next_sibling("h4")
        while next_s.text.find("Rotas Preferenciais") < 0:
            cartas.append(next_s.text)
            next_s = next_s.find_next_sibling("h4")

        # taf e metar
        taf = page_content.select_one('h5:contains("TAF")').find_next_sibling("p").text
        metar = page_content.select_one('h5:contains("METAR")').find_next_sibling("p").text

        # nascer e por do sol
        sunrise = page_content.find('sunrise').contents[0]
        sunset = page_content.find('sunset').contents[0]

        print("Aerodromo: {}".format(codigo))
        print("_____\n")
        print("CARTAS:")
        for carta in cartas:
            print("-", carta)
        print("_____\n")
        print("Nascer do sol: {}\nPor do sol   : {}".format(sunrise, sunset))
        print("_____\n")
        print("TAF  : {}\nMETAR: {}".format(taf, metar))
        print("_____\n")
    except:
        print("Algo deu errado. Tente novamente.")

    codigo = input("Digite o codigo do aerodromo ou 0 para sair:")


import MetaTrader5 as mt5
import pandas as pd
import numpy as np

class LiveTicks:
    def __init__(self,
                 c_symbol: str,
                 ):
        self.symbol = c_symbol

    def get_ticks(self, start_datetime, end_datetime):
        all_ticks = pd.DataFrame(mt5.copy_ticks_range(self.symbol, start_datetime, end_datetime, mt5.COPY_TICKS_ALL))
        return all_ticks

    def get_rates(self, window_size: int = 100, time_frame=mt5.TIMEFRAME_M1):
        all_prices = pd.DataFrame(mt5.copy_rates_from_pos(self.symbol, time_frame, 0, window_size))
        all_prices['time'] = pd.to_datetime(all_prices['time'], unit='s')
        all_prices = all_prices.set_index('time')
        all_prices = all_prices[~all_prices.index.duplicated(keep='first')]
        all_prices = all_prices.reindex(pd.date_range(all_prices.index[0], all_prices.index[-1], freq='min'))
        all_prices = all_prices.fillna(method='bfill')
        all_prices = all_prices.fillna(method='ffill')

        return all_prices

live = LiveTicks('EURUSD.si')
df = live.get_rates(window_size=300)
print(df)
# last_tick = df.index[-1]
# s_config = {
#     "window": (1 * 24 * 60),
#     "freq_modes": [4, 15, 60, 240, int((1 * 12 * 60) / 2 - 1)],
#     "point_decimal": 0.00001,
#     "slow_fast": (0.0000005, 0.01)
# }
#
# import time
# from AlgoAgent import AlgoAgent
# import matplotlib.pyplot as plt
#
# while True:
#     df = live.get_rates(window_size=24 * 60)
#
#     if last_tick < df.index[-1]:
#         last_tick = df.index[-1]
#         print(last_tick)
#         algo = AlgoAgent(df, s_config)
#         # high_modes = algo.ask_modes
#         # low_modes = algo.bid_modes
#         # plt.plot(df.index, df['close'])
#         # plt.plot(df.index,high_modes[0])
#         # plt.plot(df.index,high_modes[1])
#         # plt.plot(df.index,low_modes[0])
#         # plt.plot(df.index,low_modes[1])
#         # plt.show()
#         #
#         # plt.plot(df.index,)
#         print(algo.take_action())
#
#     time.sleep(10)

from datetime import datetime
import pytz

timezone = pytz.timezone("Etc/UTC")
utc_from = datetime(2021, 10, 10,  hour = 13,tzinfo=timezone)
utc_to = datetime(2021, 12, 11, hour = 13, tzinfo=timezone)
rates = mt5.copy_rates_range("USDJPY", mt5.TIMEFRAME_D1, utc_from, utc_to)

print(rates)


from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

from tensorflow.python.framework import graph_util
import tensorflow as tf
import argparse
import os
import sys
import tensorflow.contrib.slim as slim
import importlib
import re

def get_model_filenames(model_dir):
    files = os.listdir(model_dir)
    meta_files = [s for s in files if s.endswith('.meta')]
    if len(meta_files)==0:
        raise ValueError('No meta file found in the model directory (%s)' % model_dir)
    elif len(meta_files)>1:
        raise ValueError('There should not be more than one meta file in the model directory (%s)' % model_dir)
    meta_file = meta_files[0]
    meta_files = [s for s in files if '.ckpt' in s]
    max_step = -1
    for f in files:
        step_str = re.match(r'(^model-[\w\- ]+.ckpt-(\d+))', f)
        if step_str is not None and len(step_str.groups())>=2:
            step = int(step_str.groups()[1])
            if step > max_step:
                max_step = step
                ckpt_file = step_str.groups()[0]
    return meta_file, ckpt_file

def main(args):
    network = importlib.import_module(args.model_def)
    with tf.Graph().as_default():
        with tf.Session() as sess:
            # Load the model metagraph and checkpoint
            image_batch = tf.placeholder(tf.float32, shape=(None,args.image_size, args.image_size, 3), name='input')
            # Build the inference graph
            batch_norm_params = {
                # Decay for the moving averages
                'decay': 0.9999,
                # epsilon to prevent 0s in variance
                'epsilon': 0.001,
                # force in-place updates of mean and variance estimates
                'updates_collections': None,
                # Moving averages ends up in the trainable variables collection
                'variables_collections': [ tf.GraphKeys.TRAINABLE_VARIABLES ],
                # Only update statistics during training mode
                'is_training': False
            }
            prelogits, _ = network.inference(image_batch, args.keep_probability,
                phase_train=False, weight_decay=args.weight_decay)
            bottleneck = slim.fully_connected(prelogits, args.embedding_size, activation_fn=None,
                    weights_initializer=tf.truncated_normal_initializer(stddev=0.1),
                    weights_regularizer=slim.l2_regularizer(args.weight_decay),
                    normalizer_fn=slim.batch_norm,
                    normalizer_params=batch_norm_params,
                    scope='Bottleneck', reuse=False)
            embeddings = tf.nn.l2_normalize(bottleneck, 1, 1e-10, name='embeddings')
            print('Model directory: %s' % args.model_dir)
            _, ckpt_file = get_model_filenames(os.path.expanduser(args.model_dir))

            print('Checkpoint file: %s' % ckpt_file)

            model_dir_exp = os.path.expanduser(args.model_dir)
            all_variables = tf.get_collection_ref(tf.GraphKeys.GLOBAL_VARIABLES)
            sess.run(tf.variables_initializer(all_variables))

            saver = tf.train.Saver(
                var_list=[v for v in all_variables if "Bottleneck/BatchNorm/" not in v.name and "Bottleneck/weights" not in v.name])
            tf.get_default_session().run(tf.global_variables_initializer())
            tf.get_default_session().run(tf.local_variables_initializer())
            saver.restore(tf.get_default_session(), os.path.join(model_dir_exp, ckpt_file))

            # Retrieve the protobuf graph definition and fix the batch norm nodes
            gd = sess.graph.as_graph_def()
            for node in gd.node:
                if node.op == 'RefSwitch':
                    node.op = 'Switch'
                    for index in xrange(len(node.input)):
                        if 'moving_' in node.input[index]:
                            node.input[index] = node.input[index] + '/read'
                elif node.op == 'AssignSub':
                    node.op = 'Sub'
                    if 'use_locking' in node.attr: del node.attr['use_locking']
                elif node.op == 'AssignAdd':
                    node.op = 'Add'
                    if 'use_locking' in node.attr: del node.attr['use_locking']

            # Get the list of important nodes
            output_node_names = 'embeddings'
            whitelist_names = []
            for node in gd.node:
                if node.name.startswith('InceptionResnetV1') or node.name.startswith('embeddings') or node.name.startswith('phase_train') or node.name.startswith('Bottleneck'):
                    print(node.name)
                    whitelist_names.append(node.name)

            # Replace all the variables in the graph with constants of the same values
            output_graph_def = graph_util.convert_variables_to_constants(
                sess, gd, output_node_names.split(","),
                variable_names_whitelist=whitelist_names)

        # Serialize and dump the output graph to the filesystem
        with tf.gfile.GFile(args.output_file, 'wb') as f:
            f.write(output_graph_def.SerializeToString())
        print("%d ops in the final graph." % len(output_graph_def.node))

def parse_arguments(argv):
    parser = argparse.ArgumentParser()

    parser.add_argument('model_dir', type=str,
        help='Directory containing the metagraph (.meta) file and the checkpoint (ckpt) file containing model parameters')
    parser.add_argument('output_file', type=str,
        help='Filename for the exported graphdef protobuf (.pb)')
    parser.add_argument('--model_def', type=str,
        help='Model definition. Points to a module containing the definition of the inference graph.',
        default='models.inception_resnet_v1')
    parser.add_argument('--image_size', type=int,
        help='Image size (height, width) in pixels.', default=160)
    parser.add_argument('--keep_probability', type=float,
        help='Keep probability of dropout for the fully connected layer(s).', default=0.8)
    parser.add_argument('--weight_decay', type=float,
        help='L2 weight regularization.', default=5e-4)
    parser.add_argument('--embedding_size', type=int,
        help='Dimensionality of the embedding.', default=128)
    return parser.parse_args(argv)

if __name__ == '__main__':
    main(parse_arguments(sys.argv[1:]))


__author__ = "Chad J. Schroeder"
__copyright__ = "Copyright (C) 2005 Chad J. Schroeder"

__revision__ = "$Id$"
__version__ = "0.2"

# Standard Python modules.
import os               # Miscellaneous OS interfaces.
import sys              # System-specific parameters and functions.

# Default daemon parameters.
# File mode creation mask of the daemon.
UMASK = 0

# Default working directory for the daemon.
WORKDIR = "/"

# Default maximum for the number of available file descriptors.
MAXFD = 1024

# The standard I/O file descriptors are redirected to /dev/null by default.
if (hasattr(os, "devnull")):
   REDIRECT_TO = os.devnull
else:
   REDIRECT_TO = "/dev/null"

def createDaemon():
   """Detach a process from the controlling terminal and run it in the
   background as a daemon.
   """

   try:
      # Fork a child process so the parent can exit.  This returns control to
      # the command-line or shell.  It also guarantees that the child will not
      # be a process group leader, since the child receives a new process ID
      # and inherits the parent's process group ID.  This step is required
      # to insure that the next call to os.setsid is successful.
      pid = os.fork()
   except OSError as e:
      raise Exception("%s [%d]" % (e.strerror, e.errno))

   if (pid == 0):    # The first child.
      # To become the session leader of this new session and the process group
      # leader of the new process group, we call os.setsid().  The process is
      # also guaranteed not to have a controlling terminal.
      os.setsid()

      # Is ignoring SIGHUP necessary?
      #
      # It's often suggested that the SIGHUP signal should be ignored before
      # the second fork to avoid premature termination of the process.  The
      # reason is that when the first child terminates, all processes, e.g.
      # the second child, in the orphaned group will be sent a SIGHUP.
      #
      # "However, as part of the session management system, there are exactly
      # two cases where SIGHUP is sent on the death of a process:
      #
      #   1) When the process that dies is the session leader of a session that
      #      is attached to a terminal device, SIGHUP is sent to all processes
      #      in the foreground process group of that terminal device.
      #   2) When the death of a process causes a process group to become
      #      orphaned, and one or more processes in the orphaned group are
      #      stopped, then SIGHUP and SIGCONT are sent to all members of the
      #      orphaned group." [2]
      #
      # The first case can be ignored since the child is guaranteed not to have
      # a controlling terminal.  The second case isn't so easy to dismiss.
      # The process group is orphaned when the first child terminates and
      # POSIX.1 requires that every STOPPED process in an orphaned process
      # group be sent a SIGHUP signal followed by a SIGCONT signal.  Since the
      # second child is not STOPPED though, we can safely forego ignoring the
      # SIGHUP signal.  In any case, there are no ill-effects if it is ignored.
      #
      # import signal           # Set handlers for asynchronous events.
      # signal.signal(signal.SIGHUP, signal.SIG_IGN)

      try:
         # Fork a second child and exit immediately to prevent zombies.  This
         # causes the second child process to be orphaned, making the init
         # process responsible for its cleanup.  And, since the first child is
         # a session leader without a controlling terminal, it's possible for
         # it to acquire one by opening a terminal in the future (System V-
         # based systems).  This second fork guarantees that the child is no
         # longer a session leader, preventing the daemon from ever acquiring
         # a controlling terminal.
         pid = os.fork()    # Fork a second child.
      except OSError as e:
         raise Exception("%s [%d]" % (e.strerror, e.errno))

      if (pid == 0):    # The second child.
         # Since the current working directory may be a mounted filesystem, we
         # avoid the issue of not being able to unmount the filesystem at
         # shutdown time by changing it to the root directory.
         os.chdir(WORKDIR)
         # We probably don't want the file mode creation mask inherited from
         # the parent, so we give the child complete control over permissions.
         os.umask(UMASK)
      else:
         # exit() or _exit()?  See below.
         os._exit(0)    # Exit parent (the first child) of the second child.
   else:
      # exit() or _exit()?
      # _exit is like exit(), but it doesn't call any functions registered
      # with atexit (and on_exit) or any registered signal handlers.  It also
      # closes any open file descriptors.  Using exit() may cause all stdio
      # streams to be flushed twice and any temporary files may be unexpectedly
      # removed.  It's therefore recommended that child branches of a fork()
      # and the parent branch(es) of a daemon use _exit().
      os._exit(0)    # Exit parent of the first child.

   # Close all open file descriptors.  This prevents the child from keeping
   # open any file descriptors inherited from the parent.  There is a variety
   # of methods to accomplish this task.  Three are listed below.
   #
   # Try the system configuration variable, SC_OPEN_MAX, to obtain the maximum
   # number of open file descriptors to close.  If it doesn't exists, use
   # the default value (configurable).
   #
   # try:
   #    maxfd = os.sysconf("SC_OPEN_MAX")
   # except (AttributeError, ValueError):
   #    maxfd = MAXFD
   #
   # OR
   #
   # if (os.sysconf_names.has_key("SC_OPEN_MAX")):
   #    maxfd = os.sysconf("SC_OPEN_MAX")
   # else:
   #    maxfd = MAXFD
   #
   # OR
   #
   # Use the getrlimit method to retrieve the maximum file descriptor number
   # that can be opened by this process.  If there is not limit on the
   # resource, use the default value.
   #
   import resource        # Resource usage information.
   maxfd = resource.getrlimit(resource.RLIMIT_NOFILE)[1]
   if (maxfd == resource.RLIM_INFINITY):
      maxfd = MAXFD

   # Iterate through and close all file descriptors.
   for fd in range(0, maxfd):
      try:
         os.close(fd)
      except OSError:    # ERROR, fd wasn't open to begin with (ignored)
         pass

   # Redirect the standard I/O file descriptors to the specified file.  Since
   # the daemon has no controlling terminal, most daemons redirect stdin,
   # stdout, and stderr to /dev/null.  This is done to prevent side-effects
   # from reads and writes to the standard I/O file descriptors.

   # This call to open is guaranteed to return the lowest file descriptor,
   # which will be 0 (stdin), since it was closed above.
   os.open(REDIRECT_TO, os.O_RDWR)    # standard input (0)

   # Duplicate standard input to standard output and standard error.
   os.dup2(0, 1)            # standard output (1)
   os.dup2(0, 2)            # standard error (2)

   return(0)


import wx
from wx import adv
import os
import os.path
import subprocess
import time
import sys
from pathlib import Path
import winreg
import kh2reg

base_dir = "./"

if getattr(sys, 'frozen', False):
    base_dir = sys._MEIPASS + "/files/"

TRAY_TOOLTIP = 'PuttyAuthSock'
TRAY_ICON_INACTIVE = os.path.join(base_dir, 'icon.png')
TRAY_ICON_ACTIVE = os.path.join(base_dir, 'icon_green.png')

PLINK_CMD = 'plink.exe -ssh -A -batch -i {} {}@{} ' \
            '"rm -rf ~/.auth_sock || 0; ln -s \"$SSH_AUTH_SOCK\" \"~/.auth_sock\"; sleep infinity&; echo $!;"'

# Thanks a lot to FogleBird for getting me started on wxpython
# https://stackoverflow.com/a/6389727/891624

def subprocess_args(include_stdout=True):
    if hasattr(subprocess, 'STARTUPINFO'):
        si = subprocess.STARTUPINFO()
        si.dwFlags |= subprocess.STARTF_USESHOWWINDOW
        env = os.environ
    else:
        si = None
        env = None

    if include_stdout:
        ret = {'stdout': subprocess.PIPE}
    else:
        ret = {}

    ret.update({'stdin': subprocess.PIPE,
                'stderr': subprocess.PIPE,
                'startupinfo': si,
                'env': env })
    return ret

def create_menu_item(menu, label, func):
    item = wx.MenuItem(menu, -1, label)
    menu.Bind(wx.EVT_MENU, func, id=item.GetId())
    menu.Append(item)
    return item

class TaskBarIcon(adv.TaskBarIcon):
    def __init__(self):
        super(TaskBarIcon, self).__init__()
        self.set_icon(TRAY_ICON_INACTIVE)
        self.active = False
        self.auth_sock_pid = None
        self.Bind(wx.adv.EVT_TASKBAR_LEFT_DOWN, self.on_left_down)

    def CreatePopupMenu(self):
        menu = wx.Menu()
        # create_menu_item(menu, 'Say Hello', self.on_hello)
        # menu.AppendSeparator()
        create_menu_item(menu, 'Exit', self.on_exit)
        return menu

    def set_icon(self, path):
        icon = wx.Icon()
        icon.LoadFile(path)
        self.SetIcon(icon, TRAY_TOOLTIP)

    def on_left_down(self, event):
        if not self.active:
            self.init_auth_sock()
        else:
            self.kill_auth_sock()

    def init_auth_sock(self):
        print("Starting auth_sock")
        self.run_auth_sock()
        self.active = True
        self.set_icon(TRAY_ICON_ACTIVE)

    def run_auth_sock(self):
        ip = get_wsl_ip()
        user = get_wsl_username()
        ppk_file = get_ppk_file()
        print("Starting agent forwarding")
        process = subprocess.Popen(PLINK_CMD.format(ppk_file, user, ip), **subprocess_args(True))
        time.sleep(1)
        self.auth_sock_pid = int(process.stdout.readline().strip())

    def kill_auth_sock(self):
        print("Killing auth sock")
        kill_pid(self.auth_sock_pid)
        print("Removing auth_sock file")
        remove_auth_sock()
        self.active = False
        self.set_icon(TRAY_ICON_INACTIVE)

    def on_exit(self, event):
        if self.active:
            self.kill_auth_sock()
        wx.CallAfter(self.Destroy)

def main():
    app = wx.App()
    TaskBarIcon()
    app.MainLoop()

def get_ssh_dir():
    home = str(Path.home())
    ssh_dir = os.path.join(home, ".ssh")
    return ssh_dir

def get_ppk_file():
    ppk_file = os.path.join(get_ssh_dir(), "putty_auth_sock.ppk")
    return ppk_file

def get_wsl_ip():
    # we need to read WSL's IP address every time it has been started since it is not static
    ip_cmd = "ip -4 addr show eth0 | grep -oP '(?<=inet\s)\d+(\.\d+){3}'"
    ip = subprocess.check_output(['bash', '-c', '{}'.format(ip_cmd)], **subprocess_args(False)).decode('ascii')
    print("Using IP address {}".format(ip))
    return ip.strip()

def start_wsl_ssh():
    subprocess.check_output(['wsl', '-u', 'root', '--', 'service', 'ssh', 'start'], **subprocess_args(False))

def register_known_host():
    known_host_entry = obtain_wsl_known_host_entry()
    add_putty_reg_key(known_host_entry)

def obtain_wsl_known_host_entry():
    fingerprints = subprocess.check_output(['ssh-keyscan', get_wsl_ip()], **subprocess_args(False)).decode()
    fingerprints = fingerprints.split("\n")
    for fingerprint in fingerprints:
        if 'ssh-ed25519' in fingerprint:
            return fingerprint
    print("Error: unable to obtain ssh-rsa fingerprint")

def update_known_hosts(known_host_entry):
    known_hosts_file = os.path.join(get_ssh_dir(), "known_hosts")
    with open(known_hosts_file, 'r') as f:
        known_hosts = f.read().split("\n")

    os.remove(known_hosts_file)

    # Filter out eventually outdated known_host lines that match the fingerprint
    # known_host_entry is of format "<IP> <format> <fingerprint>"
    fingerprint = known_host_entry.split(" ")[2].strip()
    known_hosts = [known_host for known_host in known_hosts if fingerprint not in known_host]
    known_hosts.append(known_host_entry)

    with open(known_hosts_file, 'w') as f:
        f.write("\n".join([known_host.strip() for known_host in known_hosts]))

def get_wsl_username():
    username = subprocess.check_output(['bash', '-c', '"whoami"'], **subprocess_args(False)).decode('ascii')
    print("Using username {}".format(username))
    return username.strip()

def remove_auth_sock():
    subprocess.check_output(['wsl', '--', 'rm', '-rf', '~/.auth_sock'], **subprocess_args(False))

def kill_pid(pid):
    subprocess.check_output(['wsl', '--', 'kill',  '{}'.format(pid)], **subprocess_args(False))

# Adds the known_host_entry to the registry
# https://git.tartarus.org/?p=simon/putty.git;a=blob;f=contrib/kh2reg.py;hb=HEAD
def add_putty_reg_key(known_host_entry):
    regkey, regval = kh2reg.handle_line(known_host_entry)
    print(regkey)
    print(regval)
    handle = winreg.OpenKey(winreg.HKEY_CURRENT_USER, r'SOFTWARE\SimonTatham\PuTTY\SshHostKeys', 0, winreg.KEY_ALL_ACCESS)
    winreg.SetValueEx(handle, regkey, 0, winreg.REG_SZ, regval)
    winreg.CloseKey(handle)

if __name__ == '__main__':
    start_wsl_ssh()
    register_known_host()
    main()


from bs4 import BeautifulSoup
import requests
import time
from selenium import webdriver

# SCRAPING PROBLEMS
# Twitter Scraping (15pts)
# Go to your favorite follow on Twitter.  (not someone who posts explicit materials please)
# Inspect the twitter feed in Chrome.
# You'll notice that the tweets are stored in a ordered list

, and individual tweets are contained as list items

.
# Use BeautifulSoup and requests to grab the text contents of last 5 tweets located on the twitter page you chose.
# Print the tweets in a nicely formatted way.
# Have fun.  Again, nothing explicit.

#  print("{} {}!".format("Hello", "World"))
print("{} {}!".format("Hello", "World"))

twit_url = "https://twitter.com/arielhelwani"
twit_page = requests.get(twit_url)

twit_soup = BeautifulSoup(twit_page.text, "html.parser")

# tweets = twit_soup.find_all(aria_label="Timeline: Ariel Helwani’s Tweets", class_="css-1dbjc4n")
# tweet_divs = twit_soup.find_all(id="accessible-list-2")
tweet_divs = twit_soup.find_all("div", class_="tweet")
print(tweet_divs[0].text)

for i in range(5):
    print(tweet_divs[i].text)

# Weather Scraping (15pts)
# Below is a link to a 10-day weather forecast at weather.com
# Pick the weather for a city that has the first letter as your name.
# Use requests and BeautifulSoup to scrape data from the weather table.
# Print a synopsis of the weather for the next 10 days.
# Include the day and date, description, high and low temp, chance of rain, and wind. (2pts each)
# Print the weather for each of the next 10 days to the user in a readable sentences.
# You can customize the text as you like, but it should be readable as a sentence without errors. (5pts)
# You will need to target specific classes or other attributes to pull some parts of the data.
# Sample sentence:
# Wednesday, April 4 will be Partly Cloudy/Windy with a High of 37 degrees and a low of 25, humidity at 52%.  There is 0% chance of rain with winds out of the WNW at 22 mph.
# if the sentence is a little different than shown, that will work; do what you can.  Don't forget about our friend string.format()

print("{} {}!".format("Hello", "World"))

weather_url = "https://weather.com/weather/tenday/l/368d5a4e7d156c3f508cb653f95018ec2747fdacb01e8d8d17386237f84962ac"
weather_page = requests.get(weather_url)

weather_soup = BeautifulSoup(weather_page.text, "html.parser")

days = weather_soup.find_all("span", class_="day-detail clearfix")
day_list = [x.text for x in days]
print(day_list)

descriptions = weather_soup.find_all("td", class_="description")
description_list = [x.span.text for x in descriptions]
print(description_list)

his = weather_soup.find_all("td", class_="temp")
hi_list = [x.div.span.text for x in his]
print(hi_list)

lows = weather_soup.find_all("td", class_="temp")
low_list = [x.div.span.next_sibling.next_sibling.text for x in his]
print(low_list)

precips = weather_soup.find_all("td", class_="precip")
precip_list = [x.text for x in precips]
print(precip_list)

winds = weather_soup.find_all("td", class_="wind")
wind_list = [x.span.text for x in winds]
print(wind_list)

for i in range(10):
    print("\nOn {0} it will be {1} and have a max temperature of {2} degrees Fahrenheit and a min temperature of {3} degrees Fahrenheit with a {4} chance of precipitation and winds of {5}"
          .format(day_list[i],description_list[1], hi_list[i], low_list[i], precip_list[i], wind_list[i]))


import os
import logging
import fnmatch
from typing import Iterable

#
# Class to help convert from absolute path to EDK2 build path
# using workspace and packagepath variables
#

class Edk2Path(object):

    def __init__(self, ws: os.PathLike, packagepathlist: Iterable[os.PathLike], error_on_invalid_pp: bool = True):
        """ An Edk2Path object is an object that can be used to resolve edk2 relative paths

        Args:
            ws: absolute path or cwd relative path of the workspace.
            packagespathlist: list of packages path.
                Entries can be Absolute path, workspace relative path, or CWD relative.
            error_on_invalid_pp: default value is True. If packages path value is invalid raise exception
        """

        self.WorkspacePath = ws
        self.logger = logging.getLogger("Edk2Path")
        if(not os.path.isabs(ws)):
            self.WorkspacePath = os.path.abspath(os.path.join(os.getcwd(), ws))

        if(not os.path.isdir(self.WorkspacePath)):
            self.logger.error("Workspace path invalid.  {0}".format(ws))
            raise Exception("Workspace path invalid.  {0}".format(ws))

        # Set PackagePath
        self.PackagePathList = list()
        for a in packagepathlist:
            if(os.path.isabs(a)):
                self.PackagePathList.append(a)
            else:
                # see if workspace relative
                wsr = os.path.join(ws, a)
                if(os.path.isdir(wsr)):
                    self.PackagePathList.append(wsr)
                else:
                    # assume current working dir relative.  Will catch invalid dir when checking whole list
                    self.PackagePathList.append(os.path.abspath(os.path.join(os.getcwd(), a)))

        error = False
        for a in self.PackagePathList[:]:
            if(not os.path.isdir(a)):
                self.logger.log(logging.ERROR if error_on_invalid_pp else logging.WARNING,
                                "Invalid package path entry {0}".format(a))
                self.PackagePathList.remove(a)  # remove invalid path
                error = True

        # report error
        if(error and error_on_invalid_pp):
            raise Exception("Invalid package path directory(s)")

    def GetEdk2RelativePathFromAbsolutePath(self, abspath):
        ''' Given an absolute path return a edk2 path relative
        to workspace or packagespath.

        If not valid return None
        '''

        relpath = None
        found = False
        if abspath is None:
            return None
        for a in (os.path.normcase(p) for p in self.PackagePathList):
            if os.path.normcase(abspath).startswith(a):
                # found our path...now use original strings to avoid
                # change in case
                relpath = abspath[len(a):]
                found = True
                self.logger.debug("Successfully converted AbsPath to Edk2Relative Path using PackagePath")
                self.logger.debug("AbsolutePath: %s found in PackagePath: %s" % (abspath, a))
                break

        if(not found):
            # Does path start with workspace
            if os.path.normcase(abspath).startswith(os.path.normcase(self.WorkspacePath)):
                # found our path...now use original strings to avoid
                # change in case
                relpath = abspath[len(self.WorkspacePath):]
                found = True
                self.logger.debug("Successfully converted AbsPath to Edk2Relative Path using WorkspacePath")
                self.logger.debug("AbsolutePath: %s found in Workspace: %s" % (abspath, self.WorkspacePath))

        if(found):
            relpath = relpath.replace(os.sep, "/")
            return relpath.lstrip("/")

        # didn't find the path for conversion.
        self.logger.error("Failed to convert AbsPath to Edk2Relative Path")
        self.logger.error("AbsolutePath: %s" % abspath)
        return None

    def GetAbsolutePathOnThisSytemFromEdk2RelativePath(self, relpath, log_errors=True):
        ''' Given a edk2 relative path return an absolute path to the file
        in this workspace.

        Note: For case insensitive operating systems the case of the input
        relpath will be used for the return value even if it doesn't match
        the case in the filesystem.

        If not valid or doesn't exist return None'''

        if relpath is None:
            return None
        relpath = relpath.replace("/", os.sep)
        abspath = os.path.join(self.WorkspacePath, relpath)
        if os.path.exists(abspath):
            return abspath

        for a in self.PackagePathList:
            abspath = os.path.join(a, relpath)
            if(os.path.exists(abspath)):
                return abspath
        if log_errors:
            self.logger.error("Failed to convert Edk2Relative Path to an Absolute Path on this system.")
            self.logger.error("Relative Path: %s" % relpath)

        return None

    # Find the package this path belongs to using
    # some Heuristic.  This isn't perfect but at least
    # identifies the directory consistently
    #
    # @param InputPath:  absolute path to module
    #
    # @ret Name of Package that the module is in.
    def GetContainingPackage(self, InputPath):
        self.logger.debug("GetContainingPackage: %s" % InputPath)

        # Make a list that has the path case normalized for comparison.
        # This only does anything on Windows
        NormCasePackagesPathList = [os.path.normcase(x) for x in self.PackagePathList]

        # check InputPath to make sure it is at least in the folder structure of the code tree
        if os.path.normcase(self.WorkspacePath) not in os.path.normcase(InputPath):
            # not in workspace - check all the packages paths
            found_in_pp = False
            for p in NormCasePackagesPathList:
                if p in os.path.normcase(InputPath):
                    found_in_pp = True
                    break
            if(not found_in_pp):
                self.logger.error(f"{InputPath} not in code tree")
                self.logger.info("PackagePath is: %s" % os.pathsep.join(self.PackagePathList))
                self.logger.info("Workspace path is : %s" % self.WorkspacePath)
                return None

        # InputPath is in workspace or PackagesPath for worst case scenario.

        dirpathprevious = os.path.dirname(InputPath)
        dirpath = os.path.dirname(InputPath)
        for _ in range(100):  # 100 is just a counter to avoid infinite loops.  Path nodes are unlikely to exceed 100
            #
            # Check for a DEC file in this folder
            # if here then return the directory name as the "package"
            #
            for f in os.listdir(dirpath):
                if fnmatch.fnmatch(f.lower(), '*.dec'):
                    a = os.path.basename(dirpath)
                    self.logger.debug("Found DEC file at %s.  Pkg is: %s", dirpath, a)
                    return a

            #
            # if at the root of the workspace return the previous dir.
            # this catches cases where a package has no DEC
            #
            if os.path.normcase(dirpath) == os.path.normcase(self.WorkspacePath):
                a = os.path.basename(dirpathprevious)
                self.logger.debug("Reached Workspace Path.  Using previous directory: %s" % a)
                return a

            #
            # if at the root of a packagepath return the previous dir.
            # this catches cases where a package has no DEC
            #
            if os.path.normcase(dirpath) in NormCasePackagesPathList:
                a = os.path.basename(dirpathprevious)
                self.logger.debug("Reached Package Path.  Using previous directory: %s" % a)
                return a

            dirpathprevious = dirpath
            dirpath = os.path.dirname(dirpath)

        self.logger.error("Failed to find containing package for %s" % InputPath)
        self.logger.info("PackagePath is: %s" % os.pathsep.join(self.PackagePathList))
        self.logger.info("Workspace path is : %s" % self.WorkspacePath)
        return None

    # Find the list of modules (infs) that file path is in
    #
    # for now just assume any inf in the same dir or if none
    # then check parent dir.  If InputPath is not in the filesystem
    # this function will try to return the likely containing module
    # but if the entire module has been deleted this isn't possible.
    #
    # @param InputPath:  absolute path to file
    #
    # @ret list of abs path file paths for module infs
    def GetContainingModules(self, InputPath: str) -> list:
        self.logger.debug("GetContainingModules: %s" % InputPath)

        # if INF return self
        if fnmatch.fnmatch(InputPath.lower(), '*.inf'):
            return [InputPath]

        # Before checking the local filesystem for an INF
        # make sure filesystem has file or at least folder
        if not os.path.isfile(InputPath):
            logging.debug("InputPath doesn't exist in filesystem")

        modules = []
        # Check current dir
        dirpath = os.path.dirname(InputPath)
        if os.path.isdir(dirpath):
            for f in os.listdir(dirpath):
                if fnmatch.fnmatch(f.lower(), '*.inf'):
                    self.logger.debug("Found INF file in %s.  INf is: %s", dirpath, f)
                    modules.append(os.path.join(dirpath, f))

        # if didn't find any in current dir go to parent dir.
        # this handles cases like:
        # ModuleDir/
        #   Module.inf
        #   x64/
        #     file.c
        #
        if(len(modules) == 0):
            dirpath = os.path.dirname(dirpath)
            if os.path.isdir(dirpath):
                for f in os.listdir(dirpath):
                    if fnmatch.fnmatch(f.lower(), '*.inf'):
                        self.logger.debug("Found INF file in %s.  INf is: %s", dirpath, f)
                        modules.append(os.path.join(dirpath, f))

        return modules


from  rest_framework.test import APITestCase
from rest_framework.reverse import reverse
from rest_framework import status

from rest_framework_jwt.settings import api_settings

payload_handler = api_settings.JWT_PAYLOAD_HANDLER
encode_handler  = api_settings.JWT_ENCODE_HANDLER

from django.contrib.auth import get_user_model

from .models import Comment
from django.contrib.contenttypes.models import ContentType

from post.models import Post

User = get_user_model()

content_type  = ContentType.objects.get_for_model(Post)

class CommentAPITestCase(APITestCase):

	def setUp(self):

		user_obj  = User( email = 'zeus@gmail.com', full_name = 'john benjamin')
		user_obj.set_password ('oldskool123')
		user_obj.save()

		post = Post.objects.create(
			user         	=   user_obj,
			content  		= 'hello world',
			media    		= None,
			)

		comment = Comment.objects.create(
			user         		  =   user_obj,
			target       		  = content_type,
			object_id   	   	  = 1,
			content  		 	  = 'hello world',
			media    			  = None,
			)

	def test_single_user(self):
		user_obj = User.objects.count()
		self.assertEqual(user_obj, 1)

	def test_single_comment(self):
		comment  = Comment.objects.count()
		self.assertEqual(comment, 1)

	def test_get_list(self):

		data			 = {}
		url 			 = reverse('comment:comment-list')
		response 		 = self.client.get(url, data, format = 'json')
		self.assertEqual = (response.status_code, status.HTTP_200_OK)

	def test_post_comment_with_user(self):
		user_obj  = User.objects.first()
		payload   = payload_handler(user_obj)
		token_rsp = encode_handler(payload)
		print(token_rsp)
		print(user_obj)

		self.client.credentials(HTTP_AUTHORIZATION = 'JWT' + token_rsp)

		data			 = {

			'object_id'   	: 1,
			'content'  		:'My first comment',
			'media'    		:None,
			}
		url 			 = reverse('comment:comment-list')
		print(data)
		response 		 = self.client.post(url, data, format = 'json')
		self.assertEqual = (response.status_code, status.HTTP_200_OK)

	def test_put_comment_with_user(self):
		user_obj  = User.objects.first()
		payload   = payload_handler(user_obj)
		token_rsp = encode_handler(payload)
		print(token_rsp)
		print(user_obj)

		self.client.credentials(HTTP_AUTHORIZATION = 'JWT' + token_rsp)

		data			 = {

			'object_id'   	: 1,
			'content'  		:'My first comment',
			'image'    		:None,
			}
		url 			 = reverse('comment:comment-list')
		print(url)
		response 		 = self.client.post(url, data, format = 'json')
		self.assertEqual = (response.status_code, status.HTTP_200_OK)

	def test_delete_comment_with_user(self):
		user_obj  = User.objects.first()
		payload   = payload_handler(user_obj)
		token_rsp = encode_handler(payload)
		print(token_rsp)
		print(user_obj)

		self.client.credentials(HTTP_AUTHORIZATION = 'JWT' + token_rsp)

		data			 = {
			}
		post 			= Post.objects.first()
		url 			= post.get_api_url()

		response 		 = self.client.delete(url, data, format = 'json')
		self.assertEqual = (response.status_code, status.HTTP_200_OK)


# author   : Johann-Mattis List
# email    : mattis.list@lingpy.org
# created  : 2015-08-05 20:32
# modified : 2015-08-05 20:32
"""
Script prepares the data for the parsimony experiment on Tukano languages.

What this script basically does is reading in the files on sound changes and
reflexes and creating a weighted network. All data is then stored as JSON, and
JSON is again loaded by the main script that runs the analysis.
"""

__author__="Johann-Mattis List"
__date__="2015-08-05"

from lingpy import *
import networkx as nx
import json

# function to make local graph
def local_graph(changes):
    C = {}
    for line in changes:
        idx, prt, ctx = int(line[0]), line[1], line[2]
        source, target = line[-2].strip(), line[-1].strip()
        if source != target:
            try:
                C[idx, prt, ctx] += [[source, target]]
            except:
                C[idx, prt, ctx] = [[source, target]]
    return C

def global_graph(changes):
    G = nx.DiGraph()
    for line in changes:
        idx, prt, ctx = int(line[0]), line[1], line[2]
        source, target = line[-2].strip(), line[-1].strip()
        if source != target:
            G.add_edge(source, target)
    return G

def get_weight_from_graph(graph, nodeA, nodeB, chars):

    try:
        d = nx.shortest_path_length(graph, nodeA, nodeB)
    except nx.NetworkXNoPath:
        d = len(chars) * 10
    except nx.NetworkXError:
        d = len(chars) * 10

    return d

# get correspondences and changes
corrs = csv2list('D_reflexes.tsv', strip_lines=False)

# load the taxa
tdat = csv2list('D_languages.tsv', strip_lines=False)
taxa = [x[0] for x in tdat[1:]]

# get the header
header = [h.title() for h in corrs[0]]

# get the main data
data = corrs[1:]

# get the proto-data
D = {}
for line in data:
    idx = int(line[0])
    proto = line[1].strip()[1:]
    ctx = line[2]
    refs = [x.strip() for x in line]
    for i,ref in enumerate(refs):
        if '/' in ref:
            nrefs = [x.strip() for x in ref.split('/')]
        else:
            nrefs = [ref]
        refs[i] = nrefs

    tmp = dict(
            zip(
                header,
                refs
                )
            )
    tmph = sorted([h for h in tmp if h in taxa])
    patterns = [tmp[h] for h in tmph]

    D[idx, proto, ctx] = patterns

# load the two different sound change patterns
sc_complex = csv2list('D_changes.tsv')[1:]

# create the digraph
G = nx.DiGraph()

# start compiling the output dictionary to be then formatted to json
out = {}
out['patterns'] = []
out['chars'] = []
out['fitch.chars'] = []
out['fitch'] = []
out['protos'] = []

# we first make all patterns without the proto-forms
protos = sorted(D.keys())
for p in protos:

    # get the data

    pattern = D[p]
    out['patterns'] += [pattern]
    chars = []
    for px in pattern:
        for c in px:
            chars += [c]

    chars = sorted(set(chars))
    matrix = [[0 for p in chars] for c in chars]
    for i,c1 in enumerate(chars):
        for j,c2 in enumerate(chars):
            if i < j:
                matrix[i][j] = 1
                matrix[j][i] = 1
    out['fitch'] += [matrix]
    out['chars'] += [chars]
    out['fitch.chars'] += [chars]
    out['protos'] += [[p[0],p[1], p[2]]]
out['taxa'] = tmph

# we add three matrix types for now, one complete, one with the full network,
# and one with partial networks
out[''] = [] # ???
out['diwest'] = []
out['sankoff'] = []

# make the dictionary and the graph
CL = local_graph(sc_complex)

for idx, (a, b, c) in enumerate(out['protos']):

    if (a,b,c) in D:

        # create graphs for local analyses
        cdl = nx.DiGraph()
        cdl.add_edges_from(CL[a, b, c])
        cul = cdl.to_undirected()

        # all chars we want to consider this time are in the complex undirected
        # graph (but also the complex local graph)
        chars = sorted(cul.nodes())
        nlen = len(chars)

        # create the matrices
        m_cdl = [[0 for x in range(nlen)] for y in range(nlen)] # diwest model
        m_cul = [[0 for x in range(nlen)] for y in range(nlen)] # sankoff model

        for i,nA in enumerate(chars):
            for j,nB in enumerate(chars):

                m_cdl[i][j] = get_weight_from_graph(cdl, nA, nB, chars)
                m_cul[i][j] = get_weight_from_graph(cul, nA, nB, chars)

        out['diwest']    += [m_cdl]
        out['sankoff']  += [m_cul]
        out['chars'][idx] = chars
    else:
        print(a,b,c)

all_chars = []
for charset in out['chars']:
    all_chars += charset
out['allchars'] = sorted(set(all_chars))
for i,c in enumerate(out['allchars']):
    print(i+1,c)

reflexes = []
for pattern in out['patterns']:
    for p in pattern:
        reflexes += p
out['reflexes'] = sorted(set(reflexes))
for r in out['reflexes']:
    print(r)

with open('I_data.json', 'w') as f:
    f.write(json.dumps(out, indent=2))


import urllib.request, json, csv, re

#目標網址
url = "https://padax.github.io/taipei-day-trip-resources/taipei-attractions-assignment.json"

response = urllib.request.urlopen(url)
data = response.read()

data = json.loads(data)
data = data['result']['results']

#擷取區域字串函式
def get_district(str):
    result = ''
    for i in range(3):
        result += str[i]
    return result
#擷取img的第一圖檔函式
def get_imgUrl(str):
    pattern = re.compile(r"jpg|JPG")
    match = pattern.search(str)
    result = str[0:match.start()+3]
    return result

#找出所有mar資料，然後用set過濾
mrt_all = set()
for i in range(len(data)):
    if(data[i]['MRT'] == None):
        continue
    mrt_all.add(data[i]['MRT'])
#把set轉成list資料
mrt_list = list(mrt_all)
print(mrt_list)

with open("attraction.csv", mode="w", newline="") as file:
    writer = csv.writer(file)
    for i in range(len(data)):
        spot = data[i]['stitle']
        district = get_district(data[i]['address'])
        longitude = data[i]['longitude']
        imgUrl = get_imgUrl(data[i]['file'])
        writer.writerow([spot, district, longitude, imgUrl])

with open("mrt.csv", mode="w", newline="") as file:
    writer = csv.writer(file)
    for i in range(len(mrt_list)):
        row = [mrt_list[i]]
        for j in range(len(data)):
            if(data[j]['MRT'] == mrt_list[i]):
                row.append(data[j]['stitle'])
        writer.writerow(row)


# -*- coding: utf-8 -*-
"""
Combine separate, projected data sets from LAM.

Created on Wed Jan 15 10:51:31 2020
@author: artoviit
-------------------------------------------------------------------------------

Used to combine cropped data sets after each samples separate vectors have been
binned, projected and counted. For example, when each sample has been cropped
to data sets corresponding to R1-2, R3, R4-5 to provide better alignment of
samples, this script combines the sets in order to perform the rest of the
analysis as one set. The different partitions of each sample NEED to be in the
same coordinate system, i.e. they can not be rotated individually, if you want
to perform cell-to-cell calculations or clustering. For plotting and statistics
each sets individual rotation does not matter.

!!!!
When using LAM to analyse the combined data set, you need to use 'Count'
without projection; set 'project' to False. This way LAM uses the values given
by the earlier projection of the split datasets.
!!!!

Vars:
----
    data_sets : dict {int: [str, int]}:
        The data sets to combine. Key is the order in which the sets are
        combined, e.g. 1->3 from anterior to posterior. Values are the path to
        the root of LAM-hierarchical directory that contains the data set, and
        number of bins that the data set has been projected to.

    combine_chans : list [str]:
        Names of the channels that are to be combined.

    savepath : pathlib.Path:
        Path to directory where the combined data set is to be saved.

"""
import numpy as np
import pandas as pd
import pathlib as pl
import re
import shapely.geometry as gm
from shapely.ops import linemerge

# GIVE DATA SETS:
# format: {<order of combining>: [r"<path_to_dataset_root>", <bins>]}
data_sets = {1: [r"E:\Code_folder\test_ALLSTAT_split\R1R2", 7],
             2: [r"E:\Code_folder\test_ALLSTAT_split\R2R3", 24],
             3: [r"E:\Code_folder\test_ALLSTAT_split\R3R4", 7],
             4: [r"E:\Code_folder\test_ALLSTAT_split\R4R5", 15],
             5: [r"E:\Code_folder\test_ALLSTAT_split\END", 7]
             }
combine_chans = ['DAPI', 'GFP']  #'DAPIEC', 'DAPIsmall', 'GFP', 'Prospero', 'Delta']
savepath = pl.Path(r"E:\Code_folder\test_ALLSTAT_split\Combined")

def combine(path):
    """Combine data sets created by LAM."""
    # Define output-path
    fullpath = path.joinpath('Analysis Data', 'Samples')
    fullpath.mkdir(parents=True, exist_ok=True)
    # Make sure that data is in correct order
    order = sorted(data_sets.keys())
    bins = [0]  # Set first bin
    # Determine the amount to increase each sets bins
    binN = 0  # No increase for first set

    # Create list containing start bins of each set
    for ind in order[:-1]:
        binN += data_sets.get(ind)[1]
        bins.extend([binN])

    # Create list that contains paths for each dataset
    set_paths = []
    for ind in order:
        path = pl.Path(data_sets.get(ind)[0])
        samplespath = path.joinpath('Analysis Data', 'Samples')
        set_paths.append(samplespath)

    # Get names of all samples from found sample-directories
    sample_names = set([p.name for s in set_paths for p in s.iterdir() if
                        p.is_dir()])

    # Dictionary for holding vector subsets
    vectors = {s: [0] * len(data_sets.keys()) for s in sample_names}

    # COLLECT SPLIT VECTORS:
    print('Collecting vectors')
    for ind, path in enumerate(set_paths):  # Loop each dataset

        # Test found samples against total samplelist:
        sample_paths = [p for p in path.iterdir() if p.is_dir()]
        samples = set([p.name for p in sample_paths])
        diff = sample_names - samples

        if diff:  # If missing samples
            print(f"Missing samples in {data_sets.keys()[ind]}:")
            print(f"  {n}" for n in diff)

        # Gather vector data:
        vect_pat = re.compile('^vector\.csv', re.I)  # Search pattern
        for smplpath in sample_paths:
            files = [p for p in smplpath.iterdir() if p.is_file()]

            try:
                # Find, read, and store vector
                vector_path = [p for p in files if vect_pat.match(p.name)]
                vector = pd.read_csv(vector_path[0])

                # Make coordinate list of vector data
                vlist = list(zip(vector.loc[:, 'X'].astype('float'),
                             vector.loc[:, 'Y'].astype('float')))

                # Get existing data from vector dictionary and update
                vects = vectors.get(smplpath.name)
                vects[ind] = gm.LineString(vlist)
                vectors.update({smplpath.name: vects})
            except StopIteration:
                print(f"Could not find vector-file for {path.name}")

    # FIND LENGTHS OF SUB-VECTORS AND GET FULL LENGTH:
    # Loop samples and their sub-vectors
    for sample, vector_data in vectors.items():
        smplpath = fullpath.joinpath(sample)
        smplpath.mkdir(exist_ok=True)

        # Define lengths of sub-vectors
        v_lengths = [v.length for v in vector_data]

        # Summed length of sub-vectors
        full_length = sum(v_lengths)

        # Merge vectors into one and save:
        all_vects = gm.MultiLineString(vector_data) # Change to lines
        full_vector = linemerge(all_vects)  # Merge the lines
        x_d, y_d = full_vector.xy  # Coordinates of merged vector

        # Save vector as a DF
        vector_df = pd.DataFrame().assign(X=x_d, Y=y_d)
        vector_df.to_csv(smplpath.joinpath('Vector.csv'))

        # Update vector dict to have vector lengths to modify normDist
        vector_data = np.array(v_lengths) / full_length
        vectors.update({sample: vector_data})

    # MODIFYING OF CHANNEL DATA:
    # Loop all datasets
    for ind, path in enumerate(set_paths):
        print("Dataset {}: {}".format(ind+1, path.parents[1].name))
        # Get names and paths of samples
        samples = [[p.name, p] for p in path.iterdir() if p.is_dir()]

        # Loop the samples
        for smpl in samples:
            smplpath = fullpath.joinpath(smpl[0])

            # Modify channel data:
            for chan in combine_chans:  # Only modify desired channels

                # Create search pattern
                string = '^' + re.escape(chan + ".csv")
                regc = re.compile(string, re.I)

                # Match pattern
                paths = [p for p in smpl[1].iterdir() if
                         regc.fullmatch(p.name)]

                if not paths: # If no matching channels
                    print(f"No {chan} data found for {smpl[0]}")
                    continue

                # Read channel data and modify normalized distance to full set
                data = pd.read_csv(paths[0])
                data = modify_normdist(data, vectors.get(smpl[0]), ind)

                # Create savepath for data
                chan_savep = smplpath.joinpath(paths[0].name)

                # Combine channel subset with earlier data or save first set
                if ind != 0:
                    combine_chan(data, chan_savep, bins[ind])
                else:
                    data.to_csv(chan_savep, index=False)

def modify_normdist(data, fractions, f_index):
    """Adjust normalized distance of features to the full dataset."""
    if f_index == 0:  # If first in order, set to begin from zero
        min_v = 0

    # In later sets continue from distance defined by earlier sets
    else:
        min_v = sum(fractions[:f_index])

    # Define end of current set
    max_v = min_v + fractions[f_index]

    # Interpret values to the new interval
    data.NormDist = np.interp(data.NormDist, (0, 1), (min_v, max_v))
    return data

def combine_chan(data, filepath, add_bin):
    """Redefine 'ID' and 'DistBin' based on location in 'order'."""
    # Add earlier sets largest bin number to the current set for continuity
    data.DistBin = data.DistBin.apply(lambda x, b=add_bin: x+b)

    # Add earlier set's largest ID-number to current sets IDs
    try:
        org_data = pd.read_csv(filepath)
        maxID = org_data.ID.max()
        data.ID = data.ID.apply(lambda x, b=maxID: x+b)
    except FileNotFoundError:
        data.to_csv(filepath, index=False)
        return

    # Add new data to the old data and save
    org_data = pd.concat([org_data, data], ignore_index=True).reindex()
    org_data.to_csv(filepath, index=False)

if __name__ == '__main__':
    combine(savepath)
    print("DONE")


import numpy as np
import pandas as pd
import matplotlib.pyplot as plt

csvfile = "D:\\HousePrices_HalfMil.csv"
data = pd.read_csv(csvfile, index_col = None, header = None)

X = data.values[:,:15]
Y = data.values[:,15:]

X = np.array(X)
Y = np.array(Y)

X = np.concatenate([np.ones((X.shape[0],1)), X], axis=1)

w = np.linalg.solve((X.T.dot(X)), X.T.dot(Y))

y_predict = X.dot(w)

r_2 = 1 - ((Y - y_predict)**2).sum() / (((Y - y_predict.mean())**2).sum())

print(r_2)

plt.scatter(Y, y_predict)
plt.plot([Y.min(), Y.max()], [Y.min(), Y.max()], c='r')
plt.show()


from pytube import YouTube
import os
import tkinter as tk
import tkinter.ttk as ttk
import pygubu

DEFAULT_OS_LOGIN = os.getlogin()
PROJECT_PATH = os.path.abspath(os.path.dirname(__file__))
PROJECT_UI = os.path.join(PROJECT_PATH, "TubeLoader.ui")

location = (f"/home/{DEFAULT_OS_LOGIN}/Downloads")  # armazenamento

class TubeloaderApp:
    def __init__(self, master=None):
        # build ui -- user interface
        self.main_window = tk.Tk() if master is None else tk.Toplevel(master)
        self.main_window.title("TubeLoader")
        self.frame_main = tk.Frame(self.main_window)
        self.app_name = tk.Label(self.frame_main)
        self.app_name.configure(
            background='#6464f4', font='{Droid Sans Fallback} 16 {}', foreground='#ffffff', text='TubeLoader')
        self.app_name.grid(column='0', row='0')
        self.frame_main.columnconfigure('0', pad='0')
        self.entry_url = tk.Entry(self.frame_main)
        self.entry_url.configure(
            background='#becee1', font='{Arial} 12 {}', foreground='#000000', highlightbackground='#becee1')
        self.entry_url.configure(justify='center', width='60')
        _text_ = '''paste the url here'''
        self.entry_url.delete('0', 'end')
        self.entry_url.insert('0', _text_)
        self.entry_url.grid(column='0', pady='123')
        self.frame_main.columnconfigure('0', pad='0')
        self.btn_download = tk.Button(self.frame_main)
        self.btn_download.configure(
            background='#becee1', font='{Arial} 12 {}', foreground='#000000', height='1')
        self.btn_download.configure(command=lambda: self.downloader(self.entry_url.get(), location),
                                    justify='center', text='Download', width='7')
        self.btn_download.grid(column='0', pady='12', row='2')
        self.frame_main.columnconfigure('0', pad='0')
        self.Download_span = tk.Label(self.frame_main)
        self.Download_span.configure(
            background='#6464f4', font='{Arial} 12 {}', foreground='#ffff00', width='23')
        self.Download_span.grid(column='0', row='3')
        self.frame_main.configure(
            background='#6464f4', height='600', width='800')
        self.frame_main.pack(side='top')
        self.main_window.configure(
            background='#6464f4', height='200', width='200')
        self.main_window.geometry('800x600')

        # Main widget
        self.mainwindow = self.main_window

    def run(self):
        self.mainwindow.mainloop()

    def downloader(self, link, location):
        site = YouTube(link)
        highest_resolution = site.streams.get_highest_resolution()
        self.Download_span["text"] = "Downloading..."
        highest_resolution.download(location)
        self.Download_span["text"] = "Download completed!!"

if __name__ == '__main__':
    app = TubeloaderApp()
    app.run()


import cv2

# 비디오 파일 열기
video_path = 'output4.avi'
cap = cv2.VideoCapture(video_path)

if not cap.isOpened():
    print("Error: Could not open video file.")
    exit()

# 비디오 정보 가져오기
fps = int(cap.get(cv2.CAP_PROP_FPS))
frame_width = int(cap.get(cv2.CAP_PROP_FRAME_WIDTH))
frame_height = int(cap.get(cv2.CAP_PROP_FRAME_HEIGHT))

print(int(cap.get(cv2.CAP_PROP_FRAME_COUNT)))

# 프레임 추출 및 저장
frame_number = 0
while True:
    ret, frame = cap.read()
    print(ret)
    print(frame)
    if not ret:
        break

    frame_filename = f"./FrameFolder/1280x720/output4/frame_{frame_number:04d}.jpg"
    cv2.imwrite(frame_filename, frame)

    frame_number += 1

    cv2.imshow("Frame", frame)

    if cv2.waitKey(1) & 0xFF == ord('q'):
        break

# 비디오 파일 닫기
cap.release()
cv2.destroyAllWindows()


import numpy as np
from skimage.feature import hog, local_binary_pattern
from skimage.util import view_as_windows
from scipy.stats import skew, kurtosis
from sklearn import svm
from sklearn.preprocessing import StandardScaler
from sklearn.impute import SimpleImputer

import os
import cv2
import matplotlib.pyplot as plt
import sys

# from PyQt5.QtWidgets import QApplication, QGraphicsScene, QGraphicsView, QMainWindow
# from PyQt5.QtGui import QImage, QPixmap, QPainter, QPen
# from PyQt5.QtCore import Qt, QPoint

class SupportVectorMachine:
    def __init__(self, window_size=21):
        self.window_size = window_size
        self.scaler = StandardScaler()
        self.clf = svm.SVC(kernel='linear')

    def extract_features(self, img_path):

        img = cv2.imread(img_path)

        # Apply Local Binary Patterns to bgr img and extract windows
        lbp_radius = 3
        lbp_n_points = 8 * lbp_radius

        b = local_binary_pattern(img[:,:,0], lbp_n_points, lbp_radius, 'uniform')
        g = local_binary_pattern(img[:,:,1], lbp_n_points, lbp_radius, 'uniform')
        r = local_binary_pattern(img[:,:,2], lbp_n_points, lbp_radius, 'uniform')
        bgr_img = cv2.merge([b.astype(np.uint8), g.astype(np.uint8), r.astype(np.uint8)])

        # x, y, window_x, window_y, rgb
        bgr_windows = self.get_windows(bgr_img)

        # Apply Local Binary Patterns to gray img and extract windows
        gray_img = cv2.imread(img_path, cv2.IMREAD_GRAYSCALE)

        # Apply Sobel edge detection
        sobelx = cv2.Sobel(gray_img, cv2.CV_64F, 1, 0, ksize=3)
        sobely = cv2.Sobel(gray_img, cv2.CV_64F, 0, 1, ksize=3)

        # Compute magnitude and direction
        magnitude = cv2.magnitude(sobelx, sobely)
        # direction = cv2.phase(sobelx, sobely, angleInDegrees=True)

        gray_lbp = local_binary_pattern(magnitude, lbp_n_points, lbp_radius, 'uniform')

        # x, y, window_x, window_y
        gray_windows = self.get_windows(gray_lbp)

        print(gray_windows.shape)

        # hog_features = np.apply_along_axis(hog, 0, gray_lbp, orientations=8, pixels_per_cell=(4, 4), cells_per_block=(1, 1), block_norm='L2-Hys')

        # Calculate color histograms
        num_bins = 11
        gray_hist = np.array([np.histogram(gray_window, bins=num_bins)[0] for gray_window in gray_windows])
        red_hist = np.array([np.histogram(window[:,:,2], bins=num_bins, range=(0, 1))[0] for window in bgr_windows])
        green_hist = np.array([np.histogram(window[:,:,1], bins=num_bins, range=(0, 1))[0] for window in bgr_windows])
        blue_hist = np.array([np.histogram(window[:,:,0], bins=num_bins, range=(0, 1))[0] for window in bgr_windows])

        print(red_hist.shape)

        # Compute mean, median, variance, skewness, and kurtosis for each color channel histogram
        color_features = []
        for hist in [red_hist, green_hist, blue_hist, gray_hist]:
            hist_mean = np.mean(hist, axis=1)
            hist_median = np.median(hist, axis=1)
            hist_variance = np.var(hist, axis=1)
            hist_skewness = skew(hist, axis=1)
            hist_kurtosis = kurtosis(hist, axis=1)
            color_features.append(np.column_stack((hist_mean, hist_median, hist_variance, hist_skewness, hist_kurtosis)))

        # Concatenate color features
        color_features_combined = np.concatenate(color_features, axis=1)

        # Impute missing values in the color_features_combined array
        imputer = SimpleImputer(strategy='mean')
        imputed_features = imputer.fit_transform(color_features_combined)

        return self.scaler.fit_transform(imputed_features)

    def train(self, img_path, labels):
        X = self.extract_features(img_path)
        print(X.shape)
        print(labels.shape)
        self.clf.fit(X, labels)

    def predict(self, img_path):
        img = cv2.imread(img_path)
        X = self.extract_features(img_path)
        predictions = self.clf.predict(X)
        return predictions.reshape(img.shape[0], img.shape[1])

    def get_windows(self, img):

        pad_size = self.window_size // 2
        padded_image = np.pad(img, pad_width=pad_size, mode='reflect')

        if img.ndim ==3 and img.shape[-1] == 3:  # Check if the image is an RGB image (3D array)
            h, w, c = padded_image.shape
            windows = view_as_windows(padded_image, (self.window_size, self.window_size, c))
            windows = windows.reshape(-1, self.window_size, self.window_size, c)
        else:  # Handle grayscale images as before (2D array)
            windows = view_as_windows(padded_image, (self.window_size, self.window_size))
            windows = windows.reshape(-1, self.window_size, self.window_size)
        return windows

class Labeler:
    def __init__(self):
        self.draw_labeling = False
        self.last_x = None
        self.last_y = None
        self.img = None
        self.labels = None
        self.display_img = None

    def draw_line(self, x, y):
        cv2.line(self.labels, (self.last_x, self.last_y), (x, y), 1, thickness=20)
        self.last_x, self.last_y = x, y

    def label_image(self, event):
        if event.xdata and event.ydata:
            x, y = int(event.xdata), int(event.ydata)
            if self.draw_labeling:
                self.draw_line(x, y)

    def on_press(self, event):
        if event.xdata and event.ydata:
            self.draw_labeling = True
            self.last_x, self.last_y = int(event.xdata), int(event.ydata)

    def on_release(self, event):
        self.draw_labeling = False
        self.update_display()

    def update_display(self):
        self.display_img = cv2.addWeighted(self.img, 0.7, self.labels * 255, 0.3, 0)
        plt.imshow(self.display_img, cmap='gray')
        plt.draw()

    def get_labels(self, input_image_path):
        self.img = cv2.imread(input_image_path, cv2.IMREAD_GRAYSCALE)
        self.labels = np.zeros_like(self.img, dtype=np.uint8)
        fig, ax = plt.subplots()
        plt.imshow(self.img, cmap='gray')
        cid = fig.canvas.mpl_connect('button_press_event', self.on_press)
        fig.canvas.mpl_connect('motion_notify_event', self.label_image)
        fig.canvas.mpl_connect('button_release_event', self.on_release)
        plt.show()
        fig.canvas.mpl_disconnect(cid)
        return self.labels.flatten()

def overlay_labels_on_img(img_path, labels):
    img = cv2.imread(img_path, cv2.IMREAD_GRAYSCALE)
    labels = labels.reshape(img.shape[:,:,0])
    overlay_img = cv2.addWeighted(img, 0.7, labels * 255, 0.3, 0)
    return overlay_img

features_list = []
labels = []

input_image_path = "C:/Programs/Image Processing/Color Space/FeatureExtraction/test_images/T9.jpg"
labeler = Labeler()
labels = labeler.get_labels(input_image_path)

# Initialize an SVM classifier and train it with the labeled image
svm_classifier = SupportVectorMachine(window_size=21)
svm_classifier.train(input_image_path, labels)
labels = svm_classifier.predict(input_image_path)

overlay_img = overlay_labels_on_img(input_image_path, labels)

cv2.imshow('Prediction', overlay_img)

# # Load positive images (class 1)
# for filename in os.listdir(positive_image_dir):
#     if filename.endswith(".jpg") or filename.endswith(".png"):
#         # Load the image
#         img = cv2.imread(os.path.join(positive_image_dir, filename))

#         # Extract features from the image
#         features = svm_classifier.extract_features(img)

#         # Add features and label (class 1) to lists
#         features_list.append(features)
#         labels_list.append(1)

# # Load negative images (class 0)
# for filename in os.listdir(negative_image_dir):
#     if filename.endswith(".jpg") or filename.endswith(".png"):
#         # Load the image
#         img = cv2.imread(os.path.join(negative_image_dir, filename))

#         # Extract features from the image
#         features = svm_classifier.extract_features(img)

#         # Add features and label (class 0) to lists
#         features_list.append(features)
#         labels_list.append(0)

# # Convert lists to numpy arrays
# X = np.array(features_list)
# y = np.array(labels_list)

# import sys
# from PyQt5.QtWidgets import QApplication, QGraphicsScene, QGraphicsView, QMainWindow
# from PyQt5.QtGui import QImage, QPixmap, QPainter, QPen, QColor
# from PyQt5.QtCore import Qt, QPoint
# import cv2
# import numpy as np

# class InteractiveLabeling(QMainWindow):
#     def __init__(self, img):
#         super().__init__()

#         self.img = img
#         self.labels = np.zeros_like(img, dtype=np.uint8)

#         self.setWindowTitle('Interactive Labeling')

#         qimage_data = np.ndarray(shape=img.shape[::-1], dtype=np.uint8, buffer=img.data)
#         self.image = QImage(qimage_data.data, img.shape[1], img.shape[0], QImage.Format_Grayscale8)

#         self.scene = QGraphicsScene(self)
#         self.view = QGraphicsView(self.scene, self)
#         self.view.setRenderHint(QPainter.Antialiasing)
#         self.view.setRenderHint(QPainter.SmoothPixmapTransform)

#         self.pixmap = QPixmap.fromImage(self.image)
#         self.scene.addPixmap(self.pixmap)

#         self.view.setSceneRect(0, 0, self.pixmap.width(), self.pixmap.height())
#         self.setCentralWidget(self.view)

#         self.drawing = False
#         self.last_point = QPoint()

#     def mousePressEvent(self, event):
#         if event.button() == Qt.LeftButton:
#             self.drawing = True
#             self.last_point = self.view.mapToScene(event.pos()).toPoint()

#     def mouseMoveEvent(self, event):
#         if self.drawing:
#             current_point = self.view.mapToScene(event.pos()).toPoint()

#             painter = QPainter(self.pixmap)
#             painter.setPen(QPen(QColor(255, 255, 255), 10, Qt.SolidLine, Qt.RoundCap, Qt.RoundJoin))
#             painter.drawLine(self.last_point, current_point)
#             self.pixmap_item.setPixmap(self.pixmap)
#             self.last_point = current_point

#     def mouseReleaseEvent(self, event):
#         if event.button() == Qt.LeftButton:
#             self.drawing = False

#     def get_labels(self):
#         self.labels = np.array(self.pixmap.toImage().bits()).reshape(self.pixmap.height(), self.pixmap.width(), self.pixmap.depth() // 8)

#         return self.labels.ravel()

# def get_labels(input_image_path):
#     img = cv2.imread(input_image_path, cv2.IMREAD_GRAYSCALE)

#     app = QApplication(sys.argv)
#     interactive_labeling = InteractiveLabeling(img)
#     interactive_labeling.show()
#     app.exec_()

#     labels = interactive_labeling.get_labels()
#     return labels


#!python
import os
import _pickle as cPickle
import pandas as pd
import numpy as np
from sklearn.model_selection import train_test_split
from sklearn.metrics import classification_report, confusion_matrix, precision_recall_fscore_support
from sklearn.ensemble import RandomForestClassifier

train_mode=0

filesdir=os.path.dirname(os.path.abspath(__file__))
out_saved_model_file = filesdir+"/msu_hs_model.pkl"

hs_sample_file  = filesdir+"/pos_sample.csv"
nhs_sample_file = filesdir+"/neg_sample.csv"

df_hot = pd.read_csv(hs_sample_file,delimiter="\t")
df_cold = pd.read_csv(nhs_sample_file,delimiter="\t")

df_hot['sign_hs']  = True #'hot'#'hot'
df_cold['sign_hs'] = False #'no'#'cold'

print("len(df_hot)={}".format(len(df_hot)))
print("len(df_cold)={}".format(len(df_cold)))

df_all = pd.concat([df_cold,df_hot],ignore_index=True)
# df_all = df_all.dropna
df_all.dropna(inplace=True)

y_all = df_all['sign_hs'].values

######
df_all['test_i41'] =df_all['4']-df_all['4_mean']
df_all['test_iDT1']=df_all['DT']-df_all['DT_mean']
df_all['test_i43'] =df_all['4']-df_all['4_mean']-df_all['4_mad']
df_all['test_iDT3']=df_all['DT']-df_all['DT_mean']-df_all['DT_mad']
df_all['test_i53'] =df_all['5']-df_all['5_mean']-df_all['5_mad']
list_test = ['test_i41','test_iDT1','test_i43','test_iDT3','test_i53','2','4','4_mean','4_mad','5','5_mean','5_mad','DT','DT_mean','DT_mad','BkgFire_mad'] #'day',

if 'list_test' in locals():
    print("small test mode {}".format(list_test))
    X_all = df_all[list_test].values.astype('float32')
else:
    X_all = df_all.drop('sign_hs',axis=1).values.astype('float32')
    list_test = df_all.drop('sign_hs',axis=1).columns
    print("all features test mode {}".format(list_test))

X_train, X_test, y_train, y_test = train_test_split(X_all, y_all, test_size=0.3, random_state=42)
# X_train, X_test, y_train, y_test = train_test_split(X_all, y_all, test_size=0.5, random_state=42)
# X_train = X_test = X_all
# y_train = y_test = y_all

classifier = RandomForestClassifier(random_state=42)
if(train_mode==1):
    print("\nstart fit")
    classifier.fit(X_train, y_train)
else:
    with open(out_saved_model_file, 'rb') as fid:
        classifier = cPickle.load(fid)

print("start predict (only to test)")
y_predict = classifier.predict(X_test)
tn, fp, fn, tp = confusion_matrix(y_test,y_predict).ravel()
print( "confusion_matrix:\n{} (tp)\t{} (fn)\n{} (fp)\t{} (tn)".format(tp, fn, fp, tn) )
print( "\nclassification_report:\n", classification_report(y_test, y_predict, target_names=['no','hot']) )
# precision,recall,fscore,support = map(lambda x: round(x[0]*1000)/10, precision_recall_fscore_support(y_test,y_predict,average=None,labels=['hot']))
# print( "{} {} {} {}".format(precision,recall,fscore,support))

# save the classifier
if(train_mode==1):
    with open(out_saved_model_file, 'wb') as fid:
        cPickle.dump(classifier, fid)
print("finish\n")


#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.

import pecan
from pecan import core
from pecan import rest
from wsme import types as wtypes
import wsmeext.pecan as wsme_pecan

from solum.api.controllers.camp.v1_1.datamodel import (parameter_definitions
                                                       as model)
from solum.api.handlers.camp import parameter_definitions_handler
from solum.common import exception

class ParamsDefController(rest.RestController):
    """Manages operations on a singular parameter_definitions resource."""

    def __init__(self, param_defs_name):
        super(ParamsDefController, self).__init__()
        self._id = param_defs_name

    @exception.wrap_wsme_controller_exception
    @wsme_pecan.wsexpose(model.ParameterDefinitions)
    def get(self):
        """Return the appropriate parameter_definitions resource."""
        handler = (parameter_definitions_handler.
                   ParameterDefinitionsHandler(pecan.request.security_context))
        raw_defs = handler.get(self._id)
        if not raw_defs:
            core.abort(404,
                       '%s is not a parameter_definitions collection' %
                       self._id)
        host_url = pecan.request.application_url.rstrip('/')
        return raw_defs.fix_uris(host_url)

class ParameterDefinitionsController(rest.RestController):
    """Manages operations on CAMP's parameter_definitions resources."""

#    @pecan.expose()
#    def _lookup(self, param_defs_name, *remainder):
#        if remainder and not remainder[-1]:
#            remainder = remainder[:-1]
#        return ParamsDefController(param_defs_name), remainder

    @exception.wrap_wsme_controller_exception
    @wsme_pecan.wsexpose(model.ParameterDefinitions, wtypes.text)
    def get_one(self, param_defs_name):
        """Return the appropriate format resource."""
        handler = (parameter_definitions_handler.
                   ParameterDefinitionsHandler(pecan.request.security_context))
        raw_defs = handler.get(param_defs_name)
        if not raw_defs:
            core.abort(404,
                       '%s is not a parameter_definitions collection' %
                       param_defs_name)
        host_url = pecan.request.application_url.rstrip('/')
        return raw_defs.fix_uris(host_url)


#!/usr/bin/env python

"""
Package: nav_to_people

Este nodo escucha a 3 tf,un mapa (o world), una fija y otra movil, que se parametrizan mediante
el launch. Creara un topico con el modulo de la velocidad.
Tambien crea un TF que se oriente hacia la camara, adaptado para enviar los 'pose'
o destinos de la navegacion correctamente.
Requiere especialmente de navegacion_Funciones, creado en el mismo paquete.

Creator: Enrique Ortega, from Universidad de Jaen

ANTES PONIA movename[10], ahora pone movename[-1:]
"""

import rospy
import tf
import std_msgs.msg
import navegacion_Funciones as nF

if __name__ == '__main__':
    rospy.init_node('velocity_tf_module')
    fixedname = rospy.get_param('~fixframe')
    movename = rospy.get_param('~movframe')
    #rospy.loginfo("Iniciando nodo de velocidad de humano.")
    fixedframe = "/" + fixedname
    moveframe = "/" + movename
    #rospy.loginfo(fixedframe+moveframe+ " Ahora sin palito " + fixedname+ movename )
    newname = "person_new_" + movename[-1:]
    newtopic = "human_" + movename[-1:] + "_speed"

    listener = tf.TransformListener()
    br = tf.TransformBroadcaster()
    human_vel = rospy.Publisher(newtopic, std_msgs.msg.Float64, queue_size=10) #Cambiar para que sea parametrizable segun numero
    rate = rospy.Rate(20.0)

    while not rospy.is_shutdown():
        try:
            now = rospy.Time.now()
            #lookupTwist(tracking_frame, observation_frame, time, averaging_interval) linear, angular

            velocidad = nF.velocidad_humano(moveframe, fixedframe, now, 0.4, listener)
            human_vel.publish(std_msgs.msg.Float64(velocidad))
            #(linear, angular)=listener.lookupTwist(moveframe, fixedframe, rospy.Time(0), rospy.Duration(0.4)) #para calcular la velocidad de la persona
            #resultado=math.sqrt(math.pow(linear[0], 2) + math.pow(linear[1], 2))
            #rospy.loginfo(str(resultado))

            yaw_rad = nF.yaw(moveframe, fixedframe, now, listener, True)
            (roll, pitch, Z) = nF.trans_human_camera(moveframe, now, listener)

            rotacion = tf.transformations.quaternion_from_euler(-roll, -pitch, yaw_rad)
            br.sendTransform((0.0, 0.0, -Z),
                     (rotacion[0], rotacion[1], rotacion[2], rotacion[3]),
                     rospy.Time.now(),
                     newname,
                     movename)

            #final_trans = transformaciones_tf2(moveframe, "/robot_link", now, listener)

            #rospy.Subscriber("SwissRanger/people_track/coords_centroids", CoordXYZArray, calculo_angulo_tf, movename, newname)
            rate.sleep()
        except (tf.LookupException, tf.ConnectivityException, tf.ExtrapolationException, rospy.ROSInterruptException):
            pass