from Xlib import display
from time import sleep
import math
from math import atan2, degrees, pi
import subprocess
import numpy
import random
import sys
import argparse

import colormath
from colormath.color_objects import XYZColor, sRGBColor, xyYColor, HSVColor
from colormath.color_conversions import convert_color

parser = argparse.ArgumentParser(description='Choose different patterns for the lightbar.')

parser.add_argument('-d', '--delay', dest='delay', nargs='?',
                   type=float, default=2.3, const=2.3,
                   help='Delay in seconds for each color change')

parser.add_argument('-s', '--setting', dest='setting', nargs='?',
                   type=str, default='cycle', const='cycle', choices=['random', 'cycle'],
                   help='Choose between different settings')

args = parser.parse_args()
delay = args.delay
setting = args.setting

def main():
	#run chosen setting

	subprocess.Popen("ectool lightbar seq stop", shell=True)	#stop lightbar from normal seq

	if setting == 'random':
		bounceHSV()
	elif setting == 'cycle':
		circleHSV()
	else:
		exit()	#should not be able to reach this

def pol2cart(rho, phi):
    x = rho * numpy.cos(phi)
    y = rho * numpy.sin(phi)
    return(x, y)

def setColor(hexValue):		#set LEDs to passed in hex value
	subprocess.Popen("ectool lightbar 4 " + hexValue[1:3] + ' ' + hexValue[5:] + ' ' + hexValue[3:5], shell=True)

def circleHSV():	#loop along outer edge of HSV colors
	deg = random.randint(0, 359)		#start at degree 0
	increment = .6

	while 1:
		if deg > 360: deg = 0
		c = colormath.color_objects.HSVColor(deg, 1, .9)
		r = convert_color(c, sRGBColor)
		hexColor = r.get_rgb_hex()	#get hex value for color
		setColor(hexColor)

		deg += increment

		sleep(delay)

def bounceHSV():	#bounce around the HSV circle

	startX = -1
	startY = 0

	xInc = 0.01  #Amount x value will change each tick

	currX = 0
	currY = 0

	#  H = angle,	S = distance from center

	while 1:
		deg = random.randint(0, 360)	#calculate where on circle to move to
		endX, endY = pol2cart(1, math.radians(deg))	#coord to move line to
		slope = (startY - endY) / (startX - endX)

		dX = math.copysign(xInc, slope)

		currX = startX
		currY = startY
		while 1:	#move current color until position is at or past endX
			sleep(delay)
			currX += dX
			currY += (slope * xInc)

			distCenter = math.hypot(currX-0, currY-0)

			if distCenter > .99: #check if outside of unit circle
				startX = endX
				startY = endY
				break

			dx = currX - 0
			dy = currY - 0
			rads = atan2(-dy,dx)
			rads %= 2*pi
			hValue = degrees(rads)

			c = colormath.color_objects.HSVColor(hValue, distCenter, .9)
			r = convert_color(c, sRGBColor)
			hexColor = r.get_rgb_hex()

			setColor(hexColor)

main()


from datetime import datetime

from src.model.reservation_config import ReservationConfigV1
from src.model.campground_reservation_config import CampgroundReservationConfigV1
from src.model.notification_preference_config import NotificationPreferenceConfig
from src.model.auto_book_preference_config import AutoBookPreferenceConfig
from src.model.enum.sensitivity_level import SensitivityLevel

class ReservationConfigFactory:
    __date_format: str = "%m/%d/%Y"

    def get_reservation_config(self, user_config: dict) -> ReservationConfigV1:
        campgrounds = []
        for campground in user_config["campgrounds"]:
            campgrounds.append(self._get_campground_config(campground))

        result = ReservationConfigV1(
            owner_id=user_config["owner"],
            subscribers=user_config["subscribers"],
            auto_book_credentials="",
            campgrounds=campgrounds,
            permits=[],
        )

        return result

    def _get_campground_config(self, config: dict) -> CampgroundReservationConfigV1:
        result = CampgroundReservationConfigV1(
            campground_id=config["campgroundId"],
            check_in_date=datetime.strptime(config["checkInDate"], self.__date_format),
            check_out_date=datetime.strptime(config["checkOutDate"], self.__date_format),
            allow_rv_like_sites=config["allowRvLikeSites"],
            notification_preferences=self._get_notification_preference(config["notificationPreferences"]),
            auto_book_preferences=self._get_auto_book_preferences(config["autoBookPreferences"]),
        )

        return result

    def _get_notification_preference(self, config: dict) -> NotificationPreferenceConfig:
        result = NotificationPreferenceConfig(
            notifications_enabled=config["notificationsEnabled"],
            notification_sensitivity_level=SensitivityLevel[config["notificationSensitivityLevel"]],
        )

        return result

    def _get_auto_book_preferences(self, config: dict) -> AutoBookPreferenceConfig:
        result = AutoBookPreferenceConfig(
            attempt_auto_book=config["attemptAutoBook"],
            auto_book_sensitivity_level=SensitivityLevel[config["autoBookSensitivityLevel"]],
        )

        return result


import os
import glob

from torch.backends import cudnn

from parameters import get_parameters
from data_loader import Data_Loader
from model import get_dcgan
from utils import Logger

from gan import GanTrainer, BigGanTrainer

def main(config):
    cudnn.benchmark = True

    img_size = config.img_size
    batch_size = config.batch_size
    num_iters = config.iters

    classes = None
    if config.classes is not None:
        classes = config.classes.split(',')
    elif config.dataset == 'lsun':
        raise Exception('Provide class list. Available options: bedroom_train,bridge_train,church_outdoor_train')

    data_loader = Data_Loader(config.dataset, config.data_path, img_size, batch_size, classes=classes, shuffle=True)
    logger_name = '{}_{}'.format(config.dataset, config.model)

    if config.model=='dcgan':
        from torch import nn, optim
        import torch

        sn = config.sn
        sa = config.sa
        device = torch.device(config.device)

        discriminator, generator = get_dcgan(img_size, 3, sn=sn, device = device)
        d_optimizer = optim.Adam(discriminator.parameters(), lr=0.0002, betas = (0.5, 0.999))
        g_optimizer = optim.Adam(generator.parameters(), lr=0.0002, betas = (0.5, 0.999))

        if sn:
            logger_name += '_sn'

        if sa:
            logger_name += '_sa'

        logger = Logger(logger_name + '_{}_bs_{}'.format(img_size, batch_size))
        loss = nn.BCELoss()

        trainer = GanTrainer(discriminator, generator, d_optimizer, g_optimizer, loss, loss, logger, log_iter=config.log_iter, device=device)
    elif config.model == 'biggan':
        gpus = [int(x) for x in config.gpus.split(',')]
        num_classes = data_loader.num_classes
        logger = Logger(logger_name + '_{}_bs_{}'.format(img_size, batch_size))

        trainer = BigGanTrainer(num_classes, logger, log_iter=config.log_iter, gpus=gpus)

    trainer.train(data_loader.loader(), num_iters)

if __name__ == '__main__':
    config = get_parameters()
    main(config)


from reportlab.pdfbase import pdfmetrics
from reportlab.pdfbase.ttfonts import TTFont
from reportlab.pdfgen import canvas

pdfmetrics.registerFont(
    TTFont(
        'arial',
        'arial.ttf')
)

c = canvas.Canvas("hello.pdf")
c.setFont('arial', 32)
c.drawString(10, 450, "Toned Avenue")
c.showPage()
c.save()


import argparse
import torch
from pathlib import Path
from torch import nn
from torch.optim import Adam
from torch.utils.data import DataLoader
import torch.backends.cudnn as cudnn
import util.utils as utils
from albumentations import (HorizontalFlip,
                            VerticalFlip,
                            Normalize,
                            Compose)
from util.validation import validation
from util.models import TDSNet
from util.loss import LossHybrid
from util.dataset import RoboticsDataset
from util.train_val_split import get_split
from util.config import *

def make_loader(file_names, shuffle=False, transform=None, batch_size=1, works = 4):
    return DataLoader(
        dataset=RoboticsDataset(file_names, transform=transform),
        shuffle=shuffle,
        num_workers=works,
        batch_size=batch_size,
        pin_memory=torch.cuda.is_available()
    )

def train_transform(p=1):
    return Compose([
        VerticalFlip(p=0.5),
        HorizontalFlip(p=0.5),
        Normalize(p=1)
    ], p=p)

def val_transform(p=1):
    return Compose([
        Normalize(p=1)
    ], p=p)

def main():

    """
    Training script which could obtain trained model

    Args:
        some parameters saved in ./util/config.py
        alpha: weight of class task
        beta: weight of sync task
        gama: weight of scene task
        device-ids: the device id in computer
        fold: the validation set id
        root: model saved path
        batch-size: batch size
        n-epochs: maximum epochs
        lr: learning rate
        workers: cpu kernal numbers
        model: model which utilized
        num_classes: class/segment task numbers; num_scenes class numbers saved in ./util/config.py

    Returns:
        model saved in root path.
    Raises:
        None
    """

    parser = argparse.ArgumentParser()
    arg = parser.add_argument
    arg('--alpha', default=0.2, type=float)
    arg('--beta', default=0.4, type=float)
    arg('--gama', default=0.2, type=float)
    arg('--device-ids', type=str, default='0', help='For example 0,1 to run on two GPUs')
    arg('--fold', type=int, help='fold', default=0)
    arg('--root', default='runs/debug', help='checkpoint root')
    arg('--batch-size', type=int, default=1)
    arg('--n-epochs', type=int, default=100)
    arg('--lr', type=float, default=0.0001)
    arg('--workers', type=int, default=6)
    arg('--model', type=str, default='TDSNet')
    args = parser.parse_args()

    root = Path(args.root)
    root.mkdir(exist_ok=True, parents=True)

    if args.model == 'TDSNet':
        model = TDSNet(num_classes = num_classes, num_scenes = num_scenes, pretrained=True)
    if torch.cuda.is_available():
        if args.device_ids:
            device_ids = list(map(int, args.device_ids.split(',')))
        else:
            device_ids = None
        model = nn.DataParallel(model, device_ids=device_ids).cuda()
    cudnn.benchmark = True

    loss = LossHybrid(num_classes=num_classes)

    train_file_names, val_file_names = get_split(args.fold)
    print('num train = {}, num_val = {}'.format(len(train_file_names), len(val_file_names)))
    train_loader = make_loader(train_file_names, shuffle=True, transform=train_transform(p=1),
                               batch_size=args.batch_size)
    valid_loader = make_loader(val_file_names, transform=val_transform(p=1),
                               batch_size=args.batch_size)

    valid = validation

    utils.train(
        init_optimizer=lambda lr: Adam(model.parameters(), lr=lr),
        args=args,
        model=model,
        criterion=loss,
        train_loader=train_loader,
        valid_loader=valid_loader,
        validation=valid,
        fold=args.fold,
        num_classes=num_classes,
        weights = [args.alpha, args.beta, args.gama]
    )

if __name__ == '__main__':
    main()


from typing import Optional, List
from lang.python.helper import TreeNode, print_tree

class Solution:
    def binaryTreePaths(self, root: Optional[TreeNode]) -> List[str]:
        if not root:
            return []

        stack = [root]
        stackPath = [str(root.val)]
        res = []

        while stack:
            node = stack.pop()
            path = stackPath.pop()

            if node.left is None and node.right is None:
                res.append(path)

            if node.right:
                stack.append(node.right)
                stackPath.append(path + "->" + str(node.right.val))

            if node.left:
                stack.append(node.left)
                stackPath.append(path + "->" + str(node.left.val))

        return res


from flask import Blueprint, render_template, request, flash, redirect, url_for, jsonify
from flask_login import login_required, current_user
from .models import User, Post, RSVP#, Recommend
from . import db

import numpy as np
import pandas as pd
from scipy.sparse import csr_matrix
from sklearn.metrics.pairwise import cosine_similarity

views = Blueprint("views", __name__)

@views.route("/")
@views.route("/home")
@login_required
def home():
    posts = Post.query.all()
    return render_template("home.html", user=current_user, posts=posts)

@views.route("/create-post", methods=['GET', 'POST'])
@login_required
def create_post():
    if request.method == 'POST':
        text = request.form.get('text')

        if not text:
            flash('Post cannot be empty', category='error')
        else:
            post = Post(text=text, author=current_user.id)
            db.session.add(post)
            db.session.commit()
            flash('Post created!', category='success')
            return redirect(url_for('views.home'))

    return render_template('create_post.html', user=current_user)

@views.route("/delete-post/<id>")
@login_required
def delete_post(id):
    post = Post.query.filter_by(id=id).first()

    if not post:
        flash('Post does not exist.', category='error')
    elif current_user.id != post.author:
        flash('You do not have permission to delete this post.', category='error')
    else:
        db.session.delete(post)
        db.session.commit()
        flash('Post deleted!', category='success')

    return redirect(url_for('views.home'))

@views.route("/posts/<username>")
@login_required
def posts(username):
    user = User.query.filter_by(username=username).first()

    if not user:
        flash('No user with that username exist.', category='error')
        return redirect(url_for('views.home'))

    posts = Post.query.filter_by(author=user.id).all()
    return render_template("posts.html", user=current_user, posts=posts, username=username)

@views.route("/rsvp/<post_id>" )#, methods=['POST'])
@login_required
def rsvp(post_id):
    post = Post.query.filter_by(id=post_id).first()
    rsvp = RSVP.query.filter_by(author=current_user.id, post_id=post_id).first()

    if not post:
        #flash('Post does not exist.', category='error')
        return jsonify({'error':'Post does not exist.'}, 400)
    elif rsvp:
        db.session.delete(rsvp)
        db.session.commit()
    else:
        rsvp = RSVP(author=current_user.id, post_id=post_id)
        db.session.add(rsvp)
        db.session.commit()

    return redirect(url_for('views.home'))
    #return jsonify({"rsvp":len(post.rsvp), "rsvped":current_user.id in map(lambda x: x.author, post.rsvp)})

'''
bermulanya model utk recommendation system
'''

# Python SQL toolkit and Object Relational Mapper
import sqlalchemy
from sqlalchemy.ext.automap import automap_base
from sqlalchemy.orm import Session
from sqlalchemy import create_engine, inspect, func

engine = create_engine('sqlite:////home/shukor/Desktop/FYP/recommendation/website/database.db')
engine.connect()
session=Session(engine)

@views.route("/recomm/<user_id>")
@login_required
def recomm(user_id):
    user = User.query.filter_by(id=user_id).first()
    rsvp_check = RSVP.query.filter_by(author=current_user.id).first()

    if not user:
        flash('No user with that username exist.', category='error')
        return redirect(url_for('views.home'))
    if not rsvp_check:
        flash('You dont have any recommendation, like any post first.', category='error')
        return redirect(url_for('views.home'))

    query = session.query(RSVP)
    rsvp = pd.read_sql(query.statement, session.bind)

    for i in range(len(rsvp)):
        rsvp['rsvp_value'] = 1

    df_rsvp = rsvp.pivot(index='author', columns='post_id', values='rsvp_value')
    df_rsvp_dummy = df_rsvp.copy().fillna(0)

    item_similarity_df = df_rsvp_dummy.corr(method='pearson')

    def get_similar_post(post_id, rsvp):
        similar_score = item_similarity_df[post_id]*(rsvp)#-2.5)
        similar_score = similar_score.sort_values(ascending=False)

        return similar_score

    query = session.query("post_id from RSVP where author ="+user_id)
    user_data = pd.read_sql(query.statement, session.bind)

    user_data_df = pd.DataFrame(data=user_data)
    for i in range(len(user_data_df)):
        user_data_df['rsvp_value'] = 1

    user_data_list = user_data_df.to_records(index=False)
    result = list(user_data_list)

    similar_post = pd.DataFrame()

    for post,rsvp in result:
        similar_post = similar_post.append(get_similar_post(post,rsvp), ignore_index=True)

    recommend_value = list(similar_post.sum())
    recommend = pd.DataFrame()

    index = similar_post.sum().sort_values(ascending=False).head(5).index
    #index = index.head()

    #query = session.query(Post)
    #post = pd.read_sql(query.statement, session.bind)
    #posts_recommend = pd.DataFrame()

    #count = 1

    #for i in index:

    #    for j in range(len(post)+1):
    #        if j == i :
    #            query = session.query("* from post where id="+str(i))
    #            post_query = pd.read_sql(query.statement, session.bind)
    #            posts_recommend = posts_recommend.append(post_query)

                #recomm = Recommend.query.filter_by(author=current_user.id, post_id = j)#.first()

                #if not recomm:
                #    recommend = Recommend(user=current_user.id, post_id=j, rank= count)
                #    db.session.add(recommend)
                #    db.session.commit()
                #else:
                #    recommend.rank= count
                #    db.session.commit()

    #    count = count+1

    #count = 0
    #posts_recommend = posts_recommend.reset_index(drop=True)

                #recommend = Recommend(post_id=posts['id'], text=posts['text'], date_created=posts['date_created'], author=posts['author'])
                #recommend = Recommend(post_id=post_query['id'], text=post_query['text'], author=post_query['author'])
                #date_created=post_query['date_created']
                #db.session.add(recommend)
                #db.session.commit()

    ## convert dataframe in sqlite
    #posts.to_sql('Recommend', engine, if_exists='append', index=False)
    #recommends = Recommend.query.filter_by(author=current_user.id)
    #return posts_recommend.to_html()
    posts = Post.query.all()
    return render_template("recommendation.html", user=current_user, posts=posts, rsvp=index, username=current_user.username)


# -- coding: utf-8 --

import requests
from bs4 import BeautifulSoup
import retrying
import datetime

project_pull_url = "https://github.com/tensorflow/tensorflow/pull/"

def get_html(url):
    headers = {
        "user-agent": "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.132 Safari/537.36"
    }
    r = requests.get(url, timeout=15, headers=headers)
    r.raise_for_status()
    r.encoding = 'utf-8'
    return r

# 判断修改md文件,若修改了则返回1，反之返回0
@retrying.retry(wait_fixed=2000)
def judge_md(pr_num):
    # 更改md文件的标记
    flag = 0
    # 获取commit详情页地址
    commit_page_html = get_html(project_pull_url+str(pr_num)+"/commits")
    commit_page_soup = BeautifulSoup(commit_page_html.text, "html5lib")
    commit_tag = commit_page_soup.find_all("a", attrs={"class": "sha btn btn-outline BtnGroup-item"})
    if commit_tag==None:
        return 0

    # commit详情页面分析
    for i in commit_tag:
        commit_detail_url = "https://github.com" + i.attrs["href"]
        commit_detail_page = get_html(commit_detail_url)
        commit_detail_soup = BeautifulSoup(commit_detail_page.text, "html5lib")
        file_info_div = commit_detail_soup.find("div", attrs={"class": "file-info flex-auto"})
        changed_file_name = file_info_div.find_all("a", attrs={"class": "link-gray-dark"})
        for j in changed_file_name:
            if ".md" in j.text:
                flag = 1

    if flag != 0:
        return 1
    else:
        return 0

# 判断修改示例文件,若修改了则返回1，反之返回0
@retrying.retry(wait_fixed=2000)
def judge_example(pr_num):
    keywords = ["example", "Example"]
    flag = 0
    # 获取PR页面
    pr_page_html = get_html(project_pull_url + str(pr_num))
    pr_page_soup = BeautifulSoup(pr_page_html.text, "html5lib")
    # PR标题
    pr_title = pr_page_soup.find("h1", attrs={"class": "gh-header-title"}).text.strip()
    for keyword in keywords:
        if keyword in pr_title:
            flag = 1

    # PR内容获取
    pr_content = pr_page_soup.find("div", attrs={"class": "edit-comment-hide js-edit-comment-hide"}).text.strip()
    for keyword in keywords:
        if keyword in pr_content:
            flag = 1

    return flag

@retrying.retry(wait_fixed=2000)
def judge_similar_pr(pr_num):
    keywords = ["same", "duplicate", "similar"]
    flag = 0
    # 获取PR页面
    pr_page_html = get_html(project_pull_url + str(pr_num))
    pr_page_soup = BeautifulSoup(pr_page_html.text, "html5lib")

    # 获取关闭时的评论内容
    close_content = pr_page_soup.find("div", attrs={"class": "TimelineItem-badge text-white bg-red"}).parent.parent.\
        find_previous_sibling().find("div", attrs={"class": "edit-comment-hide js-edit-comment-hide"}).text.strip()
    for keyword in keywords:
        if keyword in close_content:
            flag = 1
            break

    return flag


import sys
from bs4.element import Tag
import requests
import warnings
from bs4 import BeautifulSoup
import json
import random

warnings.filterwarnings('ignore')

def obter_demonstrativo_de_parcelas(session, id_formatado):
    request = session.get('https://sic.cav.receita.fazenda.gov.br/siefpar-internet/cons/api/private/parc/consulta/demonstrativo-parcela/demonstrativo?idParc=' + id_formatado)
    soup = BeautifulSoup(request.text, 'html.parser')

    return json.loads(soup.text)

def obter_demonstrativo_de_pagamentos(session, id_formatado):
    request = session.get('https://sic.cav.receita.fazenda.gov.br/siefpar-internet/cons/api/private/parc/consulta/demonstrativo-pagamento/obter-demonstrativo-pagamentos?idParc=' + id_formatado)
    soup = BeautifulSoup(request.text, 'html.parser')

    return json.loads(soup.text)

def executar():
    cookiecav = sys.argv[1]
    aspsession = sys.argv[2]
    cnpj = sys.argv[3]

    with requests.Session() as session:
        session.cookies.set('COOKIECAV', cookiecav)
        session.cookies.set('ASP.NET_SessionId', aspsession)
        session.headers.update(
            {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36'})

        parcelamentos = []

        request = session.get('https://sic.cav.receita.fazenda.gov.br/siefpar-internet/home.html')
        soup = BeautifulSoup(request.text, 'html.parser')

        headers = {
            'Connection' : 'keep-alive',
            'sec-ch-ua': '"Chromium";v="94", "Google Chrome";v="94", ";Not A Brand";v="99"',
            'Accept' : 'application/json, text/plain, */*',
            'Content-Type' : 'application/json',
            'sec-ch-ua-mobile' : '?0',
            'sec-ch-ua-platform' : 'Windows',
            'Origin' : 'https://sic.cav.receita.fazenda.gov.br',
            'Sec-Fetch-Site' : 'same-origin',
            'Sec-Fetch-Mode' : 'cors',
            'Sec-Fetch-Dest' : 'empty',
            'Referer' : 'https://sic.cav.receita.fazenda.gov.br/siefpar-internet/ng/consultar-parcelamento/consultar-parcelamento',
            'Accept-Language' : 'pt-BR,pt;q=0.9',
        }
        data = '{"ni":"' + cnpj + '","mesInicial":null,"anoInicial":null,"mesFinal":null,"anoFinal":null}'
        request = session.post('https://sic.cav.receita.fazenda.gov.br/siefpar-internet/cons/api/private/parc/consulta/consultar-parcelamento/consultar', data = data, headers = headers)

        soup = BeautifulSoup(request.text, 'html.parser')
        if 'Não existem parcelamentos com os critérios informados' in request.text:
            return ''
        response=''
        try:
            response = json.loads(soup.text)
        except:
            pass
        if response:
            for parcelamentos in response:
                for parcelamento in parcelamentos:
                    if 'idFormatado' in parcelamento:
                        parcelamento['demonstrativo_de_parcelas'] = obter_demonstrativo_de_parcelas(session, parcelamento['idFormatado'])
                        parcelamento['demonstrativo_de_pagamentos'] = obter_demonstrativo_de_pagamentos(session, parcelamento['idFormatado'])
        session.close()
        return (json.dumps(parcelamentos))
print(executar())


#############
## Imports ##
#############

import numpy as np
import matplotlib.pyplot as plt
from scipy.signal import convolve2d

##############
## Settings ##
##############
plt.rcParams['toolbar'] = 'None'
plt.rcParams['keymap.xscale'] = 'None'
plt.rcParams['keymap.yscale'] = 'None'
plt.rcParams['axes.grid']=True

############
## Player ##
############

class Player(object):
    def __init__(self, idx, name=None, color=None):
        self.idx = idx # A player knows of himself if he is player 1 or player 2.
        self.name = 'Player %i'%idx if name is None else name
        self.color = {1:'C0',2:'C1'}[idx] if color is None else color
    def play(self, array):
        ''' The player makes a decision where to do his next move according to the provided array '''
        pass

class Person(Player):
    def play(self, array):
        try:
            x = input('%s: Where do you want to play?\t'%self.name)
            return int(x)
        except ValueError:
            return self.play(array)

class Ai(Player):
    def play(self, array):
        return int(np.random.rand()*array.shape[0])

##########
## Game ##
##########

class Game(object):
    ''' The connect 4 "board game" '''
    def __init__(self, X=7, Y=6, first=1, players=(), verbose=True):
        '''
        Create a connect 4 game.

        args:
            X = 7: size in X-dimension
            Y = 6: size in Y-dimension
            first = 1: which player is starting
            players = (): a tuple containing the two players

        '''
        self.X, self.Y = X,Y
        self.turn = first # Which player will start [1,2]
        self.array = np.zeros((X, Y), dtype=int)
        self.players = (Ai(1), Ai(2)) if len(players) != 2 else players
        self.verbose = verbose

    def get_player(self, idx):
        ''' Get player with index idx '''
        return self.players[idx - 1]

    @property
    def current_player(self):
        ''' Get current player '''
        return self.get_player(self.turn)

    def switch_player(self):
        ''' switch player '''
        self.turn = {1:2, 2:1}[self.turn]

    def play(self):
        ''' Updates board location according the the x-position specified by a player '''
        while self.state() == 0:
            self.one_turn()
        self.visualize()
        if self.verbose: print('%s WON'%self.get_player(self.state()).name)

    def one_turn(self):
        success = False
        while not success:
            self.visualize()
            x = self.current_player.play(self.array)
            success = self.update_array(self.current_player, x)
        self.switch_player() # switch player

    def update_array(self, player, x):
        ''' update array according to who played and the x position of the play '''
        y = np.sum(self.array[x] > 0)
        if y >= self.array.shape[1]: # Move not allowed
            return 0 # fail
        self.array[x,y] = player.idx
        return 1

    def visualize(self):
        ''' visualize the array in a Person friendly way '''
        if self.verbose:
            print('\n\n\n')
            A = np.array(self.array.T[::-1], dtype=str)
            A[A=='0'] = '.'
            A[A=='1'] = 'O'
            A[A=='2'] = 'X'
            for row in A:
                row = ' '.join(list(row))
                print(row)

    def state(self):
        ''' Determines the state of the board. 0: undecided; 1: player 1 won; 2: player2 won. '''
        for p in [1,2]:
            array = np.array((self.array == p), dtype=int)
            K = [np.diag(np.ones(4, dtype=int)), # diagonal 1
                     np.diag(np.ones(4, dtype=int))[::-1], # diagonal 2
                     np.ones((4,1), dtype=int), # horizontal
                     np.ones((1,4), dtype=int)] # vertical
            for k in K:
                result = convolve2d(array, k, 'valid')
                if (result == 4).any():
                    return p
        return 0

#############
## MplGame ##
#############

class MplGame(Game):
  ''' a simple connect 4 Game Visualized with matplotlib'''
  def __init__(self, X=7, Y=6, first=1, players=(), bgcolor='#ececec'):
    '''
    Create a connect 4 game implemented in matplotlib.

    args:
      X = 7: size in X-dimension
      Y = 6: size in Y-dimension
      first = 1: which player is starting
      players = (): a tuple containing the two players
      bgcolor = '#ececec': background color of the canvas

    '''
    # Initialize game
    Game.__init__(self, X, Y, first, players)

    # Matplotlib settings
    self.bgcolor = bgcolor
    self.fig, self.ax = plt.subplots()
    self.ax.set_facecolor(self.bgcolor)
    self.fig.patch.set_facecolor(self.bgcolor)
    self.fig.canvas.set_window_title('Connect 4')
    self.timer = self.fig.canvas.new_timer(interval=100)
    self.timer.add_callback(self.visualize)
    self.timer.single_shot = False

    # Initialize Game board
    self.plots = np.zeros((self.X,self.Y), dtype=object)
    for i, j in zip(*np.where(self.array == 0)):
      self.plots[i,j] = plt.scatter(i+0.5, j+0.5, s=1000, color=self.bgcolor)

    self.ax.set_title("%s's turn"%self.current_player.name, color=self.current_player.color)
    plt.xticks(range(self.X), ['']*self.X)
    plt.yticks(range(self.Y), ['']*self.Y)
    plt.axis('scaled')
    plt.xlim(0,self.X)
    plt.ylim(0,self.Y)

    try: #try to change the icon of the window
      from PyQt5 import QtGui
      plt.get_current_fig_manager().window.setWindowIcon(QtGui.QIcon('./img/icon.ico'))
    except:
      pass # keep standard matplotlib icon

  def switch_player(self):
    self.turn = {1:2, 2:1}[self.turn]
    self.ax.set_title("%s's turn"%self.current_player.name, color=self.current_player.color)

  def play(self):
    self.timer.start()
    ai_speed = 100 #[ms]
    self.players[0].timer = self.fig.canvas.new_timer(interval=ai_speed)
    self.players[1].timer = self.fig.canvas.new_timer(interval=ai_speed)

    self.cid = None
    if type(self.players[0]) == Person:
      self.cid = self.fig.canvas.mpl_connect('button_press_event', self.play_person)
    if type(self.players[1]) == Person and self.cid is None:
      self.cid = self.fig.canvas.mpl_connect('button_press_event', self.play_person)
    if type(self.players[1]) == Ai:
      self.players[1].timer.add_callback(self.play_ai)
    if type(self.players[0]) == Ai:
      self.players[0].timer.add_callback(self.play_ai)
      self.players[0].timer.start()

    plt.show()

  def play_person(self, event):
    '''
    play_person changes the internal array and replots the result
    according to where in the canvas was clicked
    '''
    x = int(event.xdata)
    success = self.update_array(self.current_player, x)
    if success:
      self.switch_player()
    if type(self.current_player) == Ai:
      self.current_player.timer.start()

  def play_ai(self):
    '''
    on_click changes the internal array and replots the result
    according to where in the canvas was clicked
    '''
    self.current_player.timer.stop()
    x = self.current_player.play(self.array)
    success = self.update_array(self.current_player, x)
    if success:
      self.switch_player()
    if type(self.current_player) == Ai:
      self.current_player.timer.start()

  def visualize(self):
    '''
    replots the array.
    In fact, all the circles are already plotted in the background
    color. This function just changes the color of these circles.
    To the color of the respective player
    '''
    if self.state() != 0:
      player = self.get_player(self.state())
      self.ax.set_title('%s WON'%player.name, color=player.color)
      self.fig.canvas.mpl_disconnect(self.cid)
      self.timer.stop()
      self.players[0].timer.stop()
      self.players[1].timer.stop()

    for i, j in zip(*np.where(self.array == 0)):
        self.plots[i,j].set_color(self.bgcolor)
    for i, j in zip(*np.where(self.array == 1)):
      self.plots[i,j].set_color(self.get_player(1).color)
    for i, j in zip(*np.where(self.array == 2)):
      self.plots[i,j].set_color(self.get_player(2).color)
    plt.draw()


import os.path
from telegraph import Telegraph

# setting
currentFile = __file__
realPath = os.path.realpath(currentFile)
dirPath = os.path.dirname(realPath)
telegraph = Telegraph()
telegraph.create_account(short_name='file-converter')

# calibre cmd
def calibrecommand(inputt,output):
    cmd = f'ebook-convert "{inputt}" "{output}" --enable-heuristics'
    print("Command to be Executed is")
    print(cmd)
    return cmd

# fontforge cmd
def fontforgecommand(inputt,output,message):
    des = dirPath + f"/{output}"
    cdes = dirPath + f"/{message.id}-convert.pe"
    text = f'Open(\'{inputt}\')\nGenerate(\'{des}\')'
    with open(f"{message.id}-convert.pe","w") as file:
        file.write(text)
    os.system(f"chmod 777 {message.id}-convert.pe")
    #cmd = f'{fontforge} --appimage-extract-and-run -script "{cdes}"'
    cmd = f'fontforge -script "{cdes}"'
    print("Command to be Executed is")
    print(cmd)
    return cmd

# libreoffice cmd
def libreofficecommand(inputt,new):
    #cmd = f'{libreoffice} --appimage-extract-and-run --headless --convert-to "{new}" "{inputt}" --outdir "{dirPath}"'
    if inputt.split(".")[-1] == 'pdf':
        cmd = f'libreoffice --infilter=="writer_pdf_import" --headless --convert-to "{new}":"writer_pdf_Export" "{inputt}" --outdir "{dirPath}"'
    else:
        cmd = f'libreoffice --headless --convert-to "{new}" "{inputt}" --outdir "{dirPath}"'
    print("Command to be Executed is")
    print(cmd)
    return cmd

# tesseract cmd
def tesrctcommand(inputt,out):
    #cmd = f'{tesseract} --appimage-extract-and-run "{inputt}" "{output}"'
    cmd = f'tesseract "{inputt}" {out}'
    print("Command to be Executed is")
    print(cmd)
    return cmd

# ffmpeg cmd
def ffmpegcommand(inputt,output,new):
    #cmd = f'{ffmpeg} -i "{inputt}" "{output}"'
    if new in  ["mp4", "mkv", "mov"]:
        cmd = f'ffmpeg -i "{inputt}" -c copy "{output}"'
    else:
        cmd = f'ffmpeg -i "{inputt}" "{output}"'
    print("Command to be Executed is")
    print(cmd)
    return cmd

# magic cmd (imagemagic)
def magickcommand(inputt,output,new):
    #cmd = f'{magick} --appimage-extract-and-run "{inputt}" "{output}"'
    if new == "ico":
        cmd = "convert"
        slist = ["256", "128", "96", "64", "48", "32", "16"]
        for ele in slist:
           toutput = updtname(inputt,f"{ele}.png")
           tcmd = f'convert "{inputt}" -resize {ele}x{ele}\! "{toutput}"'
           os.system(tcmd)
           cmd = f'{cmd} "{toutput}"'
        cmd = f'{cmd} "{output}"'
    else:
        cmd = f'convert "{inputt}" "{output}"'
    print("Command to be Executed is")
    print(cmd)
    return cmd

# new file name
def updtname(inputt,new):
    inputt = inputt.split(".")
    inputt[-1] = new
    output = ""
    for ele in inputt:
        output = output+"."+ele
    output = output[1:]
    print(f'New Filename will be' )
    print(output)
    return output

# image info
def imageinfo(file):
    cmd = f'identify -verbose {file} > "{file}.txt"'
    os.system(cmd)

    with open(f'{file}.txt', "rb") as infile:
        info = str(infile.read())
    os.remove(f'{file}.txt')

    info = info.replace(":", ": ")
    info = info.replace("b'","")
    info = info.replace("'","")
    info = info.replace("\\n","
")

    file = file.split("downloads")[-1]
    if file[0] == '/':
       file = file[1:]
    try:
        response = telegraph.create_page(f'{file.replace("./", "")}', html_content=f"
{info}
")
    except:
        return "Error in getting Info"
    return response['url']

# video info
def videoinfo(file):
    cmd = f'ffprobe -v quiet -show_format -show_streams "{file}" > "{file}.txt"'
    print(cmd)
    os.system(cmd)
    with open(f'{file}.txt', "rb") as infile:
        info = str(infile.read())

    os.remove(f'{file}.txt')

    stream = info[10:].split("[/STREAM]")
    try:
        formats = str(stream[1])[10:-12]
    except:
        formats = ""
    stream = stream[0]

    info = formats + stream[2:]
    info = info.replace("=", "     =        ")
    info = info.replace("\\n", "
")
    info = info.replace(":", "   ")
    info = info.replace("./", "")

    file = file.split("downloads")[-1]
    if file[0] == "/":
        file = file[1:]

    try:
        response = telegraph.create_page(f'{file.replace("./", "")}', html_content=f"
{info}
")
    except:
        return "Error in getting Info"
    return response["url"]

# list beautifier
def give_name(data):
    name = ""
    for i in data:
        name += ", " + str(i)
    return name[2:]


#!/bin/python3

'''
Project Name: Read Text File, Change Military Time to Standard Time,
			  Write to a New Text File and Validate the Data
Date: June 22, 2020
Author: Olga Lazarenko
Description: the purpose of the program:
			- read csv file and convert the columns with dates expressed in military time to standard time:
			  the hours will be converted to standard time, PM/AM will be added;
			- the validation of the data will be done: the rows with errors will be removed and saved at a special file;
			- Command Line Arguments (the input file, the output file,the errors file) will be passed by the user to run the code;
			- count the rows at the initial data files, at the output file, at the errors file;
			- count the rows with errors for each field

Specification: 1)dispatching_base_num: the values should be in the form 'B00123', the first character should be a letter and the following five
				characters should be numbers;
				2)pickup_datetime: the values should be in the format "%d/%m/%Y %H:%M" (ex: '02/15/2019 16:07')
				3)dropoff_datetime: the values should be in the format "%d/%m/%Y %H:%M"
				4)PULocation: the values should be three or less characters long, only positive decimals are allowed
				5)DOLocation: the values shold be three or less characters long, only positive decimals are allowed
				6)flag: the values can be either 1 or blank;

Data Source: https://data.cityofnewyork.us/Transportation/2019-High-Volume-FHV-Trip-Records/4p5c-cbgn/data
			a sample of data was used to check if the code performes properly
'''

import os
import glob
import csv
import datetime # use the module to check the datetime format
import sys # the module will allow the user to supply the command line arguments

data_folder=sys.argv[1] #the argument/the input file passed by the user at the command line
output_file=sys.argv[2] #the output file passed as the third argument at the command line
errors_file=sys.argv[3] # the errors file passed as the forth argument at the command line

print()
print("Folder exists: " + str(os.path.exists(data_folder)))
#check if the input file exists

print()
os.stat(data_folder)
# if the input folder doesn't exists, show the problem

print()

os.chdir(data_folder)
all_Files=glob.glob('*.csv')

my_Files=[]
for f in all_Files:
  if f[0:8] =="TripData":
    my_Files.append(f)

print()
print("The data files: ")  #show what files in the folder
for f in my_Files:
	print(f)
print()

data_files=[]
a=0
while a in range(len(my_Files)):
	data_files.append(data_folder + '\\'+ my_Files[a])
	a+=1
print("The full name of the files: ")#show the full names of the files in the folder
for f in data_files:
	print(f)
print('---------------------------------------------')
#declare variables to count the rows:
initial_rows=0
output_rows=0
errors_rows=0
errors_baseID=0
errors_licenceID=0
errors_pickup=0
errors_dropoff=0
errors_PUlocation=0
errors_DOlocation=0
errors_flag=0
total_errors=0

num_loop=1 # this variable will be used to control the writing the column names to the output file, the errors file
			#this should be written to the mentioned above files once

for input_file in data_files:
	with open(input_file,'rt') as file1: #open the initial file to read from it

		with open(output_file,'a+') as file2: #create a file for the output and write to it, the next data will be appended

			with open(errors_file,'a+') as file3: #create a file for rows with errors, after reading each file, the errors will be appended
				header=file1.readline() #read the first row containing the columns name
				format_time='%m/%d/%Y %H:%M'

				if num_loop==1: # the columns name should be written to the output and errors files only once
					file2.write(header) #write the columns name to the output file
					file3.write(header) #write  the columns names to the errors file
				num_loop+=1

				for line in file1: #iterate over rows of the file

					line_list=line.split(',') #split the line/row into a list by the comma separators
					initial_rows+=1 #count the initial data rows

					licenseID=line_list[0]
					base=line_list[1] #the first item in the list,
					a=line_list[2] #pickup_time,the second item in the list
					b=line_list[3] #drop-off time,the third element in the list
					PUlocation=line_list[4] #locationID, the forth element in the list
					DOlocation=line_list[5]
					flag=line_list[6]

					'''validate the licenceID field,
					if an error is present,move the row to the errors file and take another row to validate;
					count the rows with this type of error unsing the variable errors_licenceID'''
					if len(licenseID)!=6 or licenseID[0:2]!="HV" or str.isnumeric(licenseID[2:])=="False":
						file3.write(line) # move the row to the error file
						errors_licenceID+=1  # count the number of rows with this error
						errors_rows+=1 # total rows with errors
						continue #if an error is present,skip the rest of the code in the loop and take another row to validate

					'''validate the second element in the list, the baseID;
					count the errors at his field: errors_licenceID variable '''
					if len(base)!=6 or base[0]!= "B" or str.isnumeric(base[1:])=="False":
						file3.write(line) # move the row to the error file
						errors_baseID+=1 # count the errors at this field
						errors_rows+=1
						continue #if an error is present,skip the rest of the code in the loop and take another row to validate

					try: #validate the pickup date and time format
						datetime.datetime.strptime(line_list[2], format_time)
					except:
						file3.write(line) # if the datetime format is incorrect, write the row to the errors file
						errors_pickup+=1 # count the errors at this field
						errors_rows+=1
						continue

					try: #validate the drop-off  date and time format
						datetime.datetime.strptime(line_list[3],format_time)

					except:
						file3.write(line) #move the row to the error file
						errors_dropoff+=1 #count the errors at the drop-off datatime field
						errors_rows+=1
						continue
					# create the function validate_location(locationID) to validate the PUlocation/ DOlocation
					def validate_location(locationID):
						if not locationID.isnumeric():
							file3.write(line)
						elif int(locationID) <= 0:
							file3.write(line)
						elif int(locationID) >= 1000:
							file3.write(line)

					# call the function to validate pick-up/ drop-off location
					validate_location(PUlocation)
					validate_location(DOlocation)

					#validate the next column
					if flag.strip() != '':
						file3.write(line)
						errors_flag+=1 #count the rows with errors at the flag field
						errors_rows+=1
						continue

					hour_a=int(a[-5:-3])
					hour_b=int(b[-5:-3])

					def convert_to_stand_time(hour_a,hour_b): # create a function with two parameters
						if hour_a>12:
							new_hour_a=str(hour_a-12)+ a[-3:]+" PM"  # convert to standard time and add "AM/PM" to hours
						elif hour_a<12:
							new_hour_a=str(hour_a) + a[-3:]+" AM"
						elif hour_a==12:
							new_hour_a=str(hour_a) + a[-3:] + " PM"
						elif hour_a==0:
							new_hour_a='12'+ a[-3:] + " AM"

						if hour_b>12:
							new_hour_b=str(hour_b-12)+ b[-3:]+" PM"
						elif hour_b<12:
							new_hour_b=str(hour_b) + b[-3:]+" AM"
						elif hour_b==12:
							new_hour_b=str(hour_b) + b[-3:] + " PM"
						elif hour_b==0:
							new_hour_b='12'+ b[-3:] + "AM"
						return(new_hour_a, new_hour_b)

					new_hour_a, new_hour_b=convert_to_stand_time(hour_a, hour_b) # call the function 'convert_to_stand_time'

					def create_new_time(new_hour_a, new_hour_b): # the function will create new time strings for pickup and dropoff 											# pass two arguments( hour_a, hour_b) to the function
						new_a= a[0:(a.find(' ')+1)] + new_hour_a
						new_b= b[0:(b.find(' ')+1)] + new_hour_b
						return(new_a,new_b)

					new_a,new_b = create_new_time(new_hour_a,new_hour_b) # call the funtion 'create_new_time'

					def delete_insert_time(line_list): # the function will delete the 'old' time string and insert the 'new' time with AM/PM
						del line_list[2:4] # delete from the list 'old' pickup_time and dropoff_time
						line_list.insert(2,new_a) #insert into the list
						line_list.insert(3,new_b) #insert into the list
						return line_list

					line_list=delete_insert_time(line_list) # call the function
					new_str=','.join(line_list)
					file2.write(new_str) #write to the output_file
					output_rows+=1

print('The initial data (sample):')
print()
with open(data_files[1],'rt') as file1: # read the initial file
	for i in range(0,11):# print the first ten rows
		text=file1.readline()
		print(text,end='')
print()
print('-------------------------------------------')

print("My output file: {}".format(output_file))
print()
with open(output_file,'rt') as file2:
	for i in range(0,11):
		text=file2.readline() #read the first ten rows
		print(text,end='')
print('-------------------------------------------')
print()
print('The errors file (sample): {}'.format(errors_file))
print()
with open(errors_file,'rt') as file3: # read some rows from the file stored the rows with errors
	for i in range(0,11):
		text=file3.readline() #read the first ten rows
		print(text, end='')
print('------------------------------------------')
print()

print(str(initial_rows)+ "  initial data rows")
print(str(output_rows)+ "  rows at the output file")
print(str(errors_rows)+ "  rows at the errors file")
print()
print("Errors: ")
print(str(errors_licenceID)+ "   licenceID field")
print(str(errors_baseID)+ "   baseID field")
print(str(errors_pickup)+ "   pick-up date/time field")
print(str(errors_dropoff)+ "   drop-off date/time field")
print(str(errors_PUlocation)+ "   pick_up location field")
print(str(errors_DOlocation)+ "   drop-off location filed")
print(str(errors_flag)+ "   flag field")
print()


import numpy as np
from scipy.stats import norm
from ._base import GraphWorld

class DecisionTree(GraphWorld):
    """Decision tree from aversive pruning experiments.

    Attributes
    ----------
    states : array, shape = (n,)
        Indices of states.
    n_states : int
        Total number of states.
    viable_states : array
        Indices of viable states.
    n_viable_states : int
        Number of viable states.
    info : DataFrame
        Pandas DataFrame  storing the dynamics of the Markov decision process.
        Rows correspond to each viable Q-value, whereas each column contains
        its associated information.

    References
    ----------
    1. Huys, Q. J., Eshel, N., O'Nions, E., Sheridan, L., Dayan, P., & Roiser, J. P. (2012).
       Bonsai trees in your head: how the Pavlovian system sculpts goal-directed choices by
       pruning decision trees. PLoS computational biology, 8(3), e1002410.
    2. Lally, N., Huys, Q. J., Eshel, N., Faulkner, P., Dayan, P., & Roiser, J. P. (2017).
       The neural basis of aversive Pavlovian guidance during planning.
       Journal of Neuroscience, 0085-17.
    """

    def __init__(self):

        ## Define one-step transition matrix.
        T = np.ones((15,15)) * np.nan
        T[0,[ 1, 2]] = 1
        T[1,[ 3, 4]] = 1
        T[2,[ 5, 6]] = 1
        T[3,[ 7, 8]] = 1
        T[4,[ 9,10]] = 1
        T[5,[11,12]] = 1
        T[6,[13,14]] = 1
        T[np.arange(7,15),np.arange(7,15)] = 1

        ## Define rewards.
        R = np.copy(T)
        R[np.where(~np.isnan(R))] = 0
        R[0,[ 1, 2]] = [-70,-20]
        R[1,[ 3, 4]] = [-20,-70]
        R[2,[ 5, 6]] = [-20,-70]
        R[3,[ 7, 8]] = [-20, 20]
        R[4,[ 9,10]] = [ 20,140]
        R[5,[11,12]] = [-20, 20]
        R[6,[13,14]] = [-20, 20]

        ## Define start/terminal states.
        start = 0
        terminal = np.arange(7,15)

        ## Initialize GraphWorld.
        GraphWorld.__init__(self, T, R, start, terminal, epsilon=0)

    def __repr__(self):
        return '<GraphWorld | Decision Tree>'

    def _draw_nodes(self, ax, xpos, ypos, s=1000, color=None, cmap=None, vmin=None, vmax=None,
                alpha=1.0, linewidth=1.0):
        """Draw decision tree nodes. See plot_decision tree for details."""

        from matplotlib.cm import get_cmap
        from matplotlib.colors import ListedColormap, Normalize

        ## Define colors.
        if color is None:
            colors = np.repeat('#1f77b4', len(xpos))
        elif isinstance(color, str):
            colors = np.repeat(color, len(xpos))
        elif np.issubdtype(np.array(color).dtype, np.number):
            assert np.equal(len(color), len(xpos))
            if not isinstance(cmap, ListedColormap): cmap = get_cmap(cmap)
            colors = cmap(Normalize(vmin, vmax)(np.array(color)))
        else:
            assert np.equal(len(color), len(xpos))
            colors = np.copy(color)

        ## Define transparency.
        if isinstance(alpha, float):
            alphas = np.repeat(alpha, len(xpos))
        else:
            assert np.equal(len(alpha), len(xpos))
            alphas = np.copy(alpha)

        ## Iteratively plot.
        for i, (x, y, color, alpha) in enumerate(zip(xpos, ypos, colors, alphas)):
            ax.scatter(x, y, s=s, color='w', alpha=1)
            ax.scatter(x, y, s=s, color=color, alpha=alpha, linewidth=linewidth,
                       edgecolor='k')

        return ax

    def _draw_node_labels(self, ax, fontsize=14, color='w'):
        """Draws one-step reward values as node labels."""

        ## Define node positions.
        xpos = [-1,1,-1.5,-0.5,0.5,1.5,-1.75,-1.25,-0.75,-0.25,0.25,0.765,1.25,1.77]
        ypos = [1.98, 1.98, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0]

        ## Define node values.
        labels = [-70, -20, -20, -70, -20,-70, -20, 20, 20, 140, -20, 20, -20, 20]

        for x, y, label in zip(xpos, ypos, labels):
            ax.text(x, y, '%0.0f' %label, ha='center', va='center', fontsize=fontsize,
                    fontweight='heavy', color=color)

        return ax

    def _draw_edges(self, ax, xpos, ypos, edges, linewidth=1, color='0.5'):
        """Draw decision tree edges. See plot_decision tree for details."""

        ## Define line widths.
        if isinstance(linewidth, (int, float)):
            linewidth = np.repeat(linewidth, len(edges))

        ## Iteratively draw.
        for i, (s1, s2) in enumerate(edges):
            ax.plot([xpos[s1], xpos[s2]], [ypos[s1], ypos[s2]], color=color,
                    lw=linewidth[i], zorder=0)

        return ax

    def _draw_edge_labels(self, ax, labels, fontsize=14, alpha=1):

        ## Define label positions.
        xpos = [-0.6,0.6,-1.3,-0.7,0.7,1.3,-1.67,-1.33,-0.67,-0.33,0.33,0.67,1.33,1.67]
        ypos = [2.5] * 2 + [1.5] * 4 + [0.5] * 8
        halign = ['right','left'] * 7

        ## Define label transparency.
        if isinstance(alpha, (int, float)): alpha = np.repeat(alpha, len(xpos))

        ## Draw edge labels.
        for x, y, r, ha, a in zip(xpos, ypos, labels, halign, alpha):
            ax.text(x,y,'%0.0f' %r, va='center', ha=ha, fontsize=fontsize, alpha=a)

        return ax

    def _draw_path_sums(self, ax, xpos, linewidth=5, fontsize=14, alpha=1.0):

        ## Define path sums.
        sums = [-110, -70, -120, 0, -60, -20, -110, -70]

        ## Define transparency.
        if isinstance(alpha, (int, float)): alpha = np.repeat(alpha, len(sums))
        alpha = alpha[-8:]

        ## Draw line.
        ax.hlines(-0.40, -2, 2, lw=linewidth, color='k')

        ## Draw text.
        for x, r, a in zip(xpos, sums, alpha):
            ax.text(x, -0.70, '%0.0f' %r, ha='center', va='center', fontsize=fontsize, alpha=a)

        return ax

    def plot_decision_tree(self, s=1000, color=None, cmap=None, vmin=None, vmax=None, alpha=1.0,
                           node_width=1.0, node_labels=False, edge_width=1.0, edge_labels=False,
                           edge_label_alpha=1.0, path_sums=True, ax=None):
        """Plot decision tree environment.

        Parameters
        ----------
        s : float
            Node size.
        color : str
            Node color.
        cmap : matplotlib colormap name or object, or list of colors
            The mapping from data values to color space.
        vmin, vmax : floats
            Values to anchor the colormap, otherwise they are inferred
            from the data and other keyword arguments.
        alpha : float
            Node transparancy.
        node_width : float
            Width of node edges.
        node_labels : bool
            Draw node labels.
        edge_width : float
            Width of edges.
        edge_labels : bool or list
            Draw edge labels.
        edge_label_alpha : float
            Transparancy of edges.
        path_sums : bool
            Draw path sums.
        ax : matplotlib Axes
            Axes in which to draw the plot.

        Returns
        -------
        ax : matplotlib Axes
            Axes in which to draw the plot.
        """

        import matplotlib.pyplot as plt
        import seaborn as sns

        ## Initialize canvas.
        if ax is None: fig, ax = plt.subplots(1,1,figsize=(5,5))

        ## Define decision tree.
        T = np.zeros((15,15))
        T[0,[ 1, 2]] = [-70,-20]
        T[1,[ 3, 4]] = [-20,-70]
        T[2,[ 5, 6]] = [-20,-70]
        T[3,[ 7, 8]] = [-20, 20]
        T[4,[ 9,10]] = [ 20,140]
        T[5,[11,12]] = [-20, 20]
        T[6,[13,14]] = [-20, 20]

        ## Define decision tree edges.
        edges = [arr.squeeze() for arr in np.array([np.where(T)]).T]

        ## Define node positions.
        xpos = [0,-1,1,-1.5,-0.5,0.5,1.5,-1.75,-1.25,-0.75,-0.25,0.25,0.75,1.25,1.75]
        ypos = [3, 2, 2, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0]

        ## Draw DAG.
        ax = self._draw_edges(ax, xpos, ypos, edges=edges, linewidth=edge_width)
        ax = self._draw_nodes(ax, xpos, ypos, s=s, color=color, cmap=cmap, vmin=vmin, vmax=vmax,
                            alpha=alpha, linewidth=node_width)

        ## Optional details.
        if path_sums: ax = self._draw_path_sums(ax, xpos[-8:], alpha=alpha)
        if node_labels: ax = self._draw_node_labels(ax)
        if isinstance(edge_labels, (list, tuple, np.ndarray)):
            ax = self._draw_edge_labels(ax, edge_labels, alpha=edge_label_alpha)
        elif np.equal(edge_labels, True):
            ax = self._draw_edge_labels(ax, T[T.nonzero()], alpha=edge_label_alpha)

        ## Clean up.
        ax.set(xlim=(-2,2), xticks=[], yticks=[])
        sns.despine(top=True, right=True, bottom=True, left=True, ax=ax)

        return ax


from selenium import webdriver
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.common.desired_capabilities import DesiredCapabilities

def zatwierdz():
    zatwierdz = driver.find_element_by_xpath("//*[contains(text(), ' Zatwierdź ')]")
    driver.execute_script("arguments[0].click();", zatwierdz)

NFJ = 'https://nofluffjobs.com/pl'
BDJ = 'https://bulldogjob.pl/companies/jobs'
INFO = 'https://www.infopraca.pl/praca?q=junior&ct=it-programowanie-analizy&d=0&lc=Warszawa'
PATH = "C:\\Program Files\\chromedriver.exe"
chrome_options = webdriver.ChromeOptions()
chrome_options.add_experimental_option("prefs", {"profile.default_content_settings.cookies": 2})
chrome_options.add_experimental_option("prefs", {"profile.default_content_setting_values.notifications" : 2})
capa = DesiredCapabilities.CHROME
capa["pageLoadStrategy"] = "eager"
driver = webdriver.Chrome(PATH, chrome_options=chrome_options, desired_capabilities=capa)

# NoFluffJobs
driver.get(NFJ)
driver.implicitly_wait(5)
driver.find_element_by_xpath("//*[contains(text(), ' Wyrażam zgodę ')]").click()

# Lokalizacja
driver.find_element_by_xpath("//*[contains(text(), 'Lokalizacje')]").click()
driver.find_element_by_xpath("//*[contains(text(), 'Warszawa')]").click()
zatwierdz()

# Filtry
driver.find_element_by_xpath("//*[contains(text(), 'Filtry')]").click()
stazysta = driver.find_element_by_xpath("//*[contains(text(), ' Stażysta ')]")
junior = driver.find_element_by_xpath("//*[contains(text(), ' Junior ')]")
driver.execute_script("arguments[0].click();", stazysta)
driver.execute_script("arguments[0].click();", junior)
zatwierdz()

# BulldogJob
driver.execute_script(f"window.open('{BDJ}');")
driver.switch_to.window(driver.window_handles[-1])
wawa = driver.find_element_by_id('city_warszawa')
junior = driver.find_element_by_id('exp_level_junior')
driver.execute_script("arguments[0].click();", wawa)
driver.execute_script("arguments[0].click();", junior)

# InfoPraca
driver.execute_script(f"window.open('{INFO}');")
driver.switch_to.window(driver.window_handles[-1])

driver.maximize_window()

print('Good luck! ;)')


import os
import shutil
import datetime
import base64
import chardet
import codecs
from pathlib import Path
from tempfile import TemporaryDirectory, tempdir
from typing import List, Sequence, Union
from zipfile import ZipFile, ZIP_DEFLATED

from utilfuncs.common import *

def create_dir_if_not_exists(dirpath: PathLike) -> None:
    """
    Creates a directory and all intermediate directories if they don't exist

    Args:
        dirpath: Path to dir
    """
    if not dirpath.is_dir():
        os.makedirs(dirpath, exist_ok=True)

def get_base64_encoded_contents(filepath: PathLike) -> str:
    """
    Returns a b64 encoded string representing file contents

    Args:
        filepath: Path to file

    Returns:
        B64 encoded string
    """
    with open(filepath, "rb") as f:
        return base64.b64encode(f.read()).decode("utf-8")

def get_ext(filepath: PathLike) -> str:
    """
    Returns the extension of the file with prepended period (.)
    Example:
        Fetching extension::

            >>> get_ext("file.txt")
            ".txt"

    Args:
        filepath: Path of file

    Returns:
        Period prefixed extension for the given file
    """
    return "." + filepath.name.split(".")[1]

def rename_file(filepath: PathLike, new_name: str) -> PathLike:
    """
    Renames the given file with new name and returns path to renamed file.

    Args:
        filepath: Path to existing file
        new_name: New name to apply

    Returns:
        Path to renamed file
    """
    new_name_with_ext = new_name + get_ext(filepath)
    renamed_file_path = Path(filepath).parent / new_name_with_ext
    os.replace(filepath, renamed_file_path)
    return renamed_file_path

def move_file(src_file: PathLike, dest_dir: PathLike) -> PathLike:
    """
    Moves file to dest dir and returns file path in destination.

    Args:
        src_file: Path to src
        dest_dir: Path to dir

    Returns:
        Path of file in destination
    """

    destpath = Path(dest_dir) / src_file.name
    if destpath.is_file():
        os.remove(destpath)
    shutil.move(str(src_file), str(destpath))
    return destpath

def move_files(src_dir: PathLike, dest_dir: PathLike, ext="") -> None:
    """
    Moves all files of one extension from src to dest

    Args:
        src_dir: Source directory containing files to move
        dest_dir: Destination directory
        ext (str, optional): Target files to move by extension. Defaults to "".
    """
    filenames = os.listdir(src_dir)
    for name in filenames:
        filepath = Path(src_dir) / name
        if name.endswith(ext) and filepath.is_file():
            move_file(filepath, dest_dir)

def move_dir(src: PathLike, dest: PathLike) -> PathLike:
    """
    Moves dir from source to destination and returns new dir path.

    Args:
        src: Path to src
        dest: Path to new parent dir

    Returns:
        Path to directory under dest
    """
    shutil.move(str(src), str(dest))
    return Path(dest) / src.name

def move_files_matching_regex(src_dir: PathLike, dest_dir: PathLike, pattern: str) -> None:
    """
    Moves files matching regex to destination, non recursively.

    Args:
        src_dir: Path to src dir
        dest_dir: Path to dest dir
        pattern: Glob pattern
    """
    files = Path(src_dir).glob(pattern)
    for path in files:
        # skip dirs
        if not path.is_file():
            continue

        move_file(path, dest_dir)

def zipdir(dirpath: PathLike, zip_path: PathLike = None, delete_dir_afterwards=False, top_level_dir=True) -> Path:
    """
    Creates a zip archive which contains the directory specified in dirPath.

    Args:
        dirpath: Path to directory
        zip_path (PathLike, optional): Path to final zip file including name.
            If not provided, the zip file is written in dirpath's parent with the
            dirpath name and .zip extension.
        delete_dir_afterwards (bool, optional): Whether to delete the orignal dir
            after compression. Defaults to False.
        top_level_dir (bool, optional): Whether to have dirpath folder
            inside the zip or to have dirpath files directly inside the zip. Defaults to True.

    Returns:
        Path of compressed zip file
    """

    if not Path(dirpath).is_dir():
        raise InvalidDirectoryPath(f"{dirpath} is not a valid directory")

    resolved_path = Path(dirpath).expanduser().resolve(strict=True)

    final_zip_path = zip_path if zip_path is not None else (resolved_path.parent / (resolved_path.name + ".zip"))
    with ZipFile(final_zip_path, "w", ZIP_DEFLATED) as zf:
        for path in resolved_path.rglob("*"):
            rel_to_top_level = path.relative_to(resolved_path.parent) if top_level_dir else path.relative_to(resolved_path)
            zf.write(path, rel_to_top_level)

    if delete_dir_afterwards:
        shutil.rmtree(str(dirpath))

    return final_zip_path

def zipfiles(zip_path: PathLike, files_or_directories: Sequence[PathLike]) -> None:
    """
    Creates a zip archive from list of file or directory paths in destination.

    Args:
        zip_path: Path of zip file with proper name and extension
        file_paths: List of Paths
    """
    with TemporaryDirectory() as temp:

        for f in files_or_directories:
            if Path(f).is_dir():
                shutil.copytree(f, Path(temp) / f.name)
            else:
                shutil.copyfile(f, Path(temp) / f.name)

        zipdir(temp, zip_path, top_level_dir=False)

def get_contents_as_utf8(filepath: PathLike) -> str:
    """
    Returns a string representing contents of the file converted to
    utf-8.

    Args:
        filepath: Path to file

    Returns:
        File content as utf-8 string
    """
    with open(filepath, "rb") as f:
        content_bytes = f.read()
        detected = chardet.detect(content_bytes)
        file_encoding = detected["encoding"]
        # print(f"{filepath}: detected as {file_encoding}."
        if "utf-8" in file_encoding.lower():
            return filepath
        elif file_encoding is None:
            content_text = codecs.decode(content_bytes)
        else:
            content_text = codecs.decode(content_bytes, file_encoding, errors="replace")

    return codecs.encode(content_text, "utf-8", errors="replace")

def all_files_under(dirpath: PathLike, extension="") -> List[Path]:
    """
    Iterates through all files that are under the given dir with the given extension ex '.log'

    Args:
        path: The directory to scan
        extension (str, optional): The extensions to consider if provided (with period)
    Returns:
        List of Path objects or empty list if no files in directory
    """
    results = []

    try:
        currpath, _, filenames = next(os.walk(dirpath))  # only traverse one level
    except StopIteration:
        # means no files in log folder
        return results

    for name in filenames:
        if extension in name:  # empty string extension will match all files
            results.append(Path(currpath) / name)

    return results

def delete_old_files(dirpath: PathLike, days: int, extension="") -> List[Path]:
    """
    Deletes files older than a certain number of days in a given folder and returns a list of
    files that were deleted.
    Non recursive.

    Args:
        dirpath: Path for directory
        days: Age of files in days
        extension (str, optional): File extension including period prefix

    Returns:
        List of files that were deleted
    """
    num_days = -1 * days
    days_ago = datetime.datetime.now() + datetime.timedelta(num_days)

    filepaths = all_files_under(dirpath, extension)

    deleted = []

    for f in filepaths:
        last_m_time = datetime.datetime.fromtimestamp(f.stat().st_mtime)
        if last_m_time < days_ago:
            os.remove(f)
            deleted.append(f)

    return deleted


import cloudvolume
import ncollpyde
import tqdm
import pymaid

import numpy as np
import pandas as pd

from concurrent import futures

from .. import utils
from .segmentation import locs_to_segments

try:
    import mcubes
except ImportError:
    mcubes = None
except BaseException:
    raise

use_pbars = utils.use_pbars
CVtype = cloudvolume.frontends.precomputed.CloudVolumePrecomputed

__all__ = ['get_mesh', 'autoreview_edges', 'test_edges']

def get_mesh(x, bbox, vol=None):
    """Get mesh for given segmentation ID using CloudVolume.

    This produces meshes from scratch using marching cubes. As this requires
    loading the segmentation data it is not suited for generating meshes for
    whole neurons.

    Parameters
    ----------
    x :     int, list of ints
            Segmentation ID(s). Will produce a single mesh from all
            segmentation IDs.
    bbox :  list-like
            Bounding box. Either ``[x1, x2, y1, y2, z1, z2]`` or
            ``[[x1, x2], [y1, y2], [z1, z2]]``. Coordinates are expected to
            be in nanometres.
    vol :   cloudvolume.CloudVolume
            CloudVolume pointing to the segmentation data.

    Returns
    -------
    pymaid.Volume

    None
            If queried ID(s) are either only in a single voxel or if there are
            no IDs other than the queried ID(s) in the bounding box.

    """
    if not mcubes:
        raise ImportError('Unable to import mcubes (PyMCubes) library.')

    if not isinstance(x, (list, np.ndarray)):
        x = [x]

    if isinstance(vol, type(None)):
        vol = getattr(utils, 'vol')

    assert isinstance(vol, CVtype)

    if not isinstance(vol, CVtype):
        raise TypeError('Expected CloudVolume, got "{}"'.format(type(vol)))

    # Parse bbox
    bbox = np.array(bbox).reshape(3, 2)
    # Convert to voxel coords
    bbox = bbox / np.array(vol.scale['resolution']).reshape(3, 1)
    bbox = np.array([np.floor(bbox[:, 0]), np.ceil(bbox[:, 1])]).astype(int).T

    # Add some padding - otherwise we might get [:1] slices
    bbox += [0, 1]

    x1, x2, y1, y2, z1, z2 = bbox.flatten()

    # Get this cutout
    u = vol[x1:x2, y1:y2, z1:z2][:, :, :, 0]

    # Subset to the desired ID(s)
    u = np.isin(u, x)

    n_voxels = u.sum()

    if n_voxels == 0:
        raise ValueError("Id(s) {} not found in given bounding box.".format(x))
    elif n_voxels == 1:
        #  print('ID(s) are in a single voxel.')
        return None
    elif n_voxels == np.prod(u.shape):
        #  print('All voxels in cutout are queried segmentation IDs')
        return None

    # We have to add some padding otherwise the marching cube will fail
    u = np.pad(u, 1, mode='constant', constant_values=False)

    # Update bounding box to match padding
    bbox[:, 0] -= 1

    # Generate faces and vertices
    verts, faces = mcubes.marching_cubes(u, 0)

    # Bring into the correct coordinate space
    verts *= vol.scale['resolution']
    verts += bbox[:, 0] * vol.scale['resolution']

    # Turn into and return pymaid Volume
    return pymaid.Volume(verts, faces, name=str(x))

def autoreview_edges(x, conf_threshold=1, vol=None, remote_instance=None):
    """Automatically review (low-confidence) edges between nodes.

    The way this works:
      1. Fetch the live version of the neuron(s) from the CATMAID instance
      2. Use raycasting to test (low-confidence) edges
      3. Edge confidence is set to ``5`` if test is passed and to ``1`` if not

    You *can* use this function to test all edges in a neuron by increasing
    ``conf_threshold`` to 5. Please note that this could produce a lot of false
    positives (i.e. edges will be flagged as incorrect even though they
    aren't). Part of the problem is that mitochondria are segmented as
    separate entities and hence introduce membranes inside a neuron.

    Parameters
    ----------
    x :                 skeleton ID(s) | pymaid.CatmaidNeuron/List
                        Neuron(s) to review.
    conf_threshold :    int, optional
                        Confidence threshold for edges to be tested. By
                        default only reviews edges with confidence <= 1.
    vol :               cloudvolume.CloudVolume, optional
                        CloudVolume pointing to segmentation data.
    remote_instance :   pymaid.CatmaidInstance, optional
                        CATMAID instance. If ``None``, will use globally
                        define instance.

    Returns
    -------
    server response
                        CATMAID server response from updating node
                        confidences.

    See Also
    --------
    :func:`fafbseg.test_edges`
                        If you only need to test without changing confidences.

    Examples
    --------
    >>> # Set up CloudVolume from the publicly hosted FAFB segmentation data
    >>> # (if you have a local copy, use that instead)
    >>> from cloudvolume import CloudVolume
    >>> vol = CloudVolume('https://storage.googleapis.com/fafb-ffn1-20190805/segmentation',
    ...                   cache=True,
    ...                   progress=False)
    >>> # Autoreview edges
    >>> _ = fafbseg.autoreview_edges(14401884, vol=vol, remote_instance=manual)

    """
    # Fetch neuron(s)
    n = pymaid.get_neurons(x, remote_instance=remote_instance)

    # Extract low confidence edges
    not_root = ~n.nodes.parent_id.isnull()
    is_low_conf = n.nodes.confidence <= conf_threshold
    to_test = n.nodes[is_low_conf & not_root]

    if to_test.empty:
        print('No low-confidence edges to test in neuron(s) '
              '{} found'.format(n.skeleton_id))
        return

    # Test edges
    verdict = test_edges(n,
                         edges=to_test[['treenode_id', 'parent_id']].values,
                         vol=vol)

    # Update node confidences
    new_confidences = {n: 5 for n in to_test[verdict].treenode_id.values}
    new_confidences.update({n: 1 for n in to_test[~verdict].treenode_id.values})
    resp = pymaid.update_node_confidence(new_confidences,
                                         remote_instance=remote_instance)

    msg = '{} of {} tested low-confidence edges were found to be correct.'
    msg = msg.format(sum(verdict), to_test.shape[0])
    print(msg)

    return resp

def test_edges(x, edges=None, vol=None, max_workers=4):
    """Test if edge(s) cross membranes using ray-casting.

    Parameters
    ----------
    x :             pymaid.CatmaidNeuron | pandas.DataFrame
                    Neuron or treenode table to test edges for.
    edges :         list-like, optional
                    Use to subset to given edges. Can be:
                     1. List of single treenode IDs
                     2. List of pairs of treenode IDs
                     3. ``None`` in which case all edges will be tested. This
                        excludes the root node as it doesn't have an edge!
    vol :           cloudvolume.CloudVolume
                    CloudVolume pointing to segmentation data.
    max_workers :   int, optional
                    Maximum number of parallel worker processes to test edges.

    Returns
    -------
    numpy.ndarray
                (N, ) array containing True/False for each tested edge.

    See Also
    --------
    :func:`fafbseg.autoreview_edges`
                Use if you want to automatically review low confidence edges.

    """
    if isinstance(vol, type(None)):
        vol = getattr(utils, 'vol')

    assert isinstance(vol, CVtype)

    if isinstance(x, pymaid.CatmaidNeuron):
        nodes = x.nodes
    elif isinstance(x, pd.DataFrame):
        nodes = x
    else:
        raise TypeError('Expected CatmaidNeuron or DataFrame,'
                        ' got "{}"'.format(type(x)))

    if isinstance(edges, type(None)):
        not_root = ~nodes.parent_id.isnull()
        edges = nodes[not_root].treenode_id.values

    edges = np.array(edges)

    nodes = nodes.set_index('treenode_id', inplace=False)
    if edges.ndim == 1:
        locs1 = nodes.loc[edges][['x', 'y', 'z']].values
        parents = nodes.loc[edges].parent_id.values
        locs2 = nodes.loc[parents][['x', 'y', 'z']].values
    elif edges.ndim == 2:
        locs1 = nodes.loc[edges[:, 0]][['x', 'y', 'z']].values
        locs2 = nodes.loc[edges[:, 1]][['x', 'y', 'z']].values
    else:
        raise ValueError('Unexpected format for edges: {}'.format(edges.shape))

    # Get the segmentation IDs at the first location
    segids1 = locs_to_segments(locs1)
    with tqdm.tqdm(total=len(segids1), desc='Testing edges') as pbar:
        with futures.ThreadPoolExecutor(max_workers=max_workers) as ex:
            point_futures = [ex.submit(_test_single_edge, *k, vol=vol) for k in zip(locs1,
                                                                                locs2,
                                                                                segids1)]
            for f in futures.as_completed(point_futures):
                pbar.update(1)

    return np.array([f.result() for f in point_futures])

def _test_single_edge(l1, l2, seg_id, vol):
    """Test single edge.

    Parameters
    ----------
    l1, l2 :    int | float
                Locations of the nodes connected by the edge
    seg_id :    int
                Segment ID of one of the nodes.
    vol :       cloudvolume.CloudVolume

    Returns
    -------
    bool

    """
    # Get the bounding box
    bbox = np.array([l1, l2])
    bbox = np.array([bbox.min(axis=0), bbox.max(axis=0)]).T

    # Get the mesh
    mesh = get_mesh(seg_id, bbox=bbox, vol=vol)

    # No mesh means that edge is most likely True
    if not mesh:
        return True

    # Prepare raycasting
    coll = ncollpyde.Volume(np.array(mesh.vertices, dtype=float, order='C'),
                            np.array(mesh.faces, dtype=np.int32, order='C'))

    # Get intersections
    l1 = l1.reshape(1, 3)
    l2 = l2.reshape(1, 3)
    inter_ix, inter_xyz, is_inside = coll.intersections(l1, l2)

    # If not intersections treat this edge as True
    if not inter_xyz.any():
        return True

    return True


import torch
import torch.nn as nn
from torch.autograd import Variable
import click
import numpy as np
from numpy import random
import torch.nn.functional as F
from torch import optim
import random
import sys
import pickle
import pdb
import time
import os
import platform

import cv2
import matplotlib.pyplot as plt
import matplotlib.image as mpimg

from tensorboardX import SummaryWriter

# Uber-only:
# import OpusHdfsCopy
# from OpusHdfsCopy import transferFileToHdfsDir, checkHdfs

# Loading the image data. This requires downloading the CIFAR 10 dataset (Python version) - https://www.cs.toronto.edu/~kriz/cifar.html

###########       loading track data      ############
IMGSIZE = 32
OTBDIR = '../testing_dataset/OTB100/'
datafileList = os.listdir(OTBDIR)
datafileList.sort()
img = []
# for numfile in range(len(datafileList)):
for numfile in range(1):
    OTBSUBDIR = OTBDIR + datafileList[3]  # [numfile]  dataset Basketball
    rectFile = OTBSUBDIR + '/groundtruth_rect.txt'

    rectArr = []
    with open(rectFile, 'r') as file_to_read:
        while True:
            lines = file_to_read.readline()
            # print(lines,lines.type)
            if not lines:
                break
                pass
            p_tmp = [int(i) for i in lines.split(',')]
            rectArr.append(p_tmp)
        rectArr = np.array(rectArr)
        pass

    # print(rectArr)
    # print(rectArr[1])

    imgfileList = os.listdir(OTBSUBDIR + '/img')
    imgfileList.sort()

    # print(imgfileList)
    # print(len(imgfileList))

    for numfile in range(len(imgfileList)):
        picFile = OTBSUBDIR + '/img/' + imgfileList[numfile]
        pic = mpimg.imread(picFile)
        # plt.imshow(pic)
        # plt.show()
        rectImg = rectArr[numfile]
        picRect = pic[rectImg[1]:rectImg[1] + rectImg[3], rectImg[0]:rectImg[0] + rectImg[2]]
        # plt.imshow(picRect)
        # plt.show()
        picRectResize = cv2.resize(picRect, (IMGSIZE, IMGSIZE))
        # plt.imshow(picRectResize)
        # plt.show()
        # print(picRectResize,picRectResize.shape)
        # picRect2d = np.concatenate(picRect)
        picRect2d = np.concatenate(picRectResize)
        picRect1d = np.concatenate(picRect2d)
        img.append(picRect1d)
    img = np.array(img)
    print("img:")
    print(img, img.shape)

############################################################

np.set_printoptions(precision=4)

defaultParams = {
    'nbpatterns': 2,  # number of images per episode
    'nbprescycles': 3,  # number of presentations for each image
    'prestime': 20,  # number of time steps for each image presentation
    'prestimetest': 3,  # number of time steps for the test (degraded) image
    'interpresdelay': 2,  # number of time steps (with zero input) between two presentations
    'patternsize': 1024,  # size of the images (32 x 32 = 1024)
    'nbiter': 100000,  # number of episodes
    'probadegrade': .5,
    # when contiguousperturbation is False (which it shouldn't be), probability of zeroing each pixel in the test image
    'lr': 1e-4,  # Adam learning rate
    'print_every': 10,  # how often to print statistics and save files
    'homogenous': 0,  # whether alpha should be shared across connections
    'rngseed': 0  # random seed
}

# ttype = torch.FloatTensor;         # For CPU
ttype = torch.cuda.FloatTensor;  # For GPU

# Generate GT0 TI TI_ GTi+1
def generateGTArr(params, i, yout, Tout):  # ,Ti_,ti
    inputT = np.zeros((params['nbsteps'], 1, params['nbneur']))
    patterns = []
    GT0 = img[0].reshape((3, 1024)).sum(0).astype(float)
    GT0 = GT0[:params['patternsize']]
    GT0 = GT0 - np.mean(GT0)
    GT0 = GT0 / (1e-8 + np.max(np.abs(GT0)))
    patterns.append(GT0)
    # print("GT0")
    # print(GT0)
    # patterns.append(Ti)
    Ti = img[i].reshape((3, 1024)).sum(0).astype(float)
    Ti = Ti[:params['patternsize']]
    Ti = Ti - np.mean(Ti)
    Ti = Ti / (1e-8 + np.max(np.abs(Ti)))
    # print("Ti")
    # print(Ti)
    patterns.append(Ti)

    if Tout == False:
        _Ti = Ti
        patterns.append(_Ti)  # use ti as ti_
    else:
        _Ti = yout  # output
        patterns.append(_Ti)
    # print("_Ti")
    # print(_Ti)
    # print("pattern:")
    # print(patterns)

    GTi1 = img[i + 1].reshape((3, 1024)).sum(0).astype(float)
    GTi1 = GTi1[:params['patternsize']]
    GTi1 = GTi1 - np.mean(GTi1)
    GTi1 = GTi1 / (1e-8 + np.max(np.abs(GTi1)))
    targetpattern = GTi1.copy()

    # print("targetpattern:")
    # print(targetpattern)

    # preservedbits = np.ones(params['patternsize'])
    # Inserting the inputs in the input tensor at the proper places
    for nc in range(params['nbprescycles']):  # 3
        np.random.shuffle(patterns)
        for ii in range(params['nbpatterns']):  # 3
            for nn in range(params['prestime']):  # 20
                numi = nc * (params['nbpatterns'] * (params['prestime'])) + ii * (
                    params['prestime']) + nn
                # print("numi:")
                # print(numi)
                inputT[numi][0][:params['patternsize']] = patterns[ii][:]

    for nn in range(params['nbsteps']):
        inputT[nn][0][-1] = 1.0  # Bias neuron is forced to 1
        # inputT[nn] *= params['inputboost']       # Strengthen inputs

    inputT = torch.from_numpy(inputT).type(ttype)  # Convert from numpy to Tensor
    target = torch.from_numpy(targetpattern).type(ttype)

    return inputT, target

##################################

def generateTrainArr(params, yout, Tout):  # ,Ti_,ti
    inputT = np.zeros((params['nbsteps'], 1, params['nbneur']))
    patterns = []
    GT0 = img[0].reshape((3, 1024)).sum(0).astype(float)
    GT0 = GT0[:params['patternsize']]
    GT0 = GT0 - np.mean(GT0)
    GT0 = GT0 / (1e-8 + np.max(np.abs(GT0)))
    patterns.append(GT0)
    # print("GT0")
    # print(GT0)
    # patterns.append(Ti)
    i = np.random.randint(img.shape[0] - 1)
    Ti = img[i].reshape((3, 1024)).sum(0).astype(float)
    Ti = Ti[:params['patternsize']]
    Ti = Ti - np.mean(Ti)
    Ti = Ti / (1e-8 + np.max(np.abs(Ti)))
    # print("Ti")
    # print(Ti)
    patterns.append(Ti)

    if Tout == False:
        _Ti = Ti
        patterns.append(_Ti)  # use ti as ti_
    else:
        _Ti = yout  # output
        patterns.append(_Ti)
    # print("_Ti")
    # print(_Ti)
    # print("pattern:")
    # print(patterns)

    GTi1 = img[i + 1].reshape((3, 1024)).sum(0).astype(float)
    GTi1 = GTi1[:params['patternsize']]
    GTi1 = GTi1 - np.mean(GTi1)
    GTi1 = GTi1 / (1e-8 + np.max(np.abs(GTi1)))
    targetpattern = GTi1.copy()

    # print("targetpattern:")
    # print(targetpattern)

    # preservedbits = np.ones(params['patternsize'])
    # Inserting the inputs in the input tensor at the proper places
    for nc in range(params['nbprescycles']):  # 3
        # np.random.shuffle(patterns)

        for ii in range(params['nbpatterns']):  # 3
            for nn in range(params['prestime']):  # 20
                numi = nc * (params['nbpatterns'] * (params['prestime'])) + ii * (
                    params['prestime']) + nn
                # print("numi:",numi)
                inputT[numi][0][:params['patternsize']] = patterns[ii][:]

    for nn in range(params['nbsteps']):
        inputT[nn][0][-1] = 1.0  # Bias neuron is forced to 1
        # inputT[nn] *= params['inputboost']       # Strengthen inputs

    inputT[0][0][:params['patternsize']] = patterns[0][:]
    inputT = torch.from_numpy(inputT).type(ttype)  # Convert from numpy to Tensor
    target = torch.from_numpy(targetpattern).type(ttype)

    return inputT, target

class Network(nn.Module):
    def __init__(self, params):
        super(Network, self).__init__()
        # Notice that the vectors are row vectors, and the matrices are transposed wrt the comp neuro order, following deep learning / pytorch conventions
        # Each *column* of w targets a single output neuron
        self.w = Variable(.01 * torch.randn(params['nbneur'], params['nbneur']).type(ttype),
                          requires_grad=True)  # fixed (baseline) weights
        if params['homogenous'] == 1:
            self.alpha = Variable(.01 * torch.ones(1).type(ttype),
                                  requires_grad=True)  # plasticity coefficients: homogenous/shared across connections
        else:
            self.alpha = Variable(.01 * torch.randn(params['nbneur'], params['nbneur']).type(ttype),
                                  requires_grad=True)  # plasticity coefficients: independent
        self.eta = Variable(.01 * torch.ones(1).type(ttype),
                            requires_grad=True)  # "learning rate" of plasticity, shared across all connections
        self.params = params

    def forward(self, input, yin, hebb):
        # Inputs are fed by clamping the output of cells that receive input at the input value, like in standard Hopfield networks
        # clamps = torch.zeros(1, self.params['nbneur'])
        clamps = np.zeros(self.params['nbneur'])
        # print("CLAMPS 1 ")
        # print(clamps,clamps.shape)

        zz = torch.nonzero(input.data[0].cpu()).numpy().squeeze()
        # print("ZZ ZZ ZZ ZZ ZZ ZZ ")
        # print(zz, zz.shape)
        clamps[zz] = 1
        # print(clamps)
        # print(1-clamps)
        # print(input)
        # print(yin)
        clamps = Variable(torch.from_numpy(clamps).type(ttype), requires_grad=False).float()

        # writer.add_image('{input}_feature_maps', input, global_step=0)
        # print("clamps:")
        # print(clamps)
        # print(1-clamps)
        yout = F.tanh(yin.mm(self.w + torch.mul(self.alpha, hebb))) * (1 - clamps) + input * clamps
        hebb = (1 - self.eta) * hebb + self.eta * torch.bmm(yin.unsqueeze(2), yout.unsqueeze(1))[0]  # bmm used to implement outer product

        # print("YIN YOUT ")
        # print(yin.shape)
        # print(yout.shape)
        # print(yin)
        # print(self.w,self.w.shape)
        # print(self.alpha,self.alpha.shape)
        # print(F.tanh(yin.mm(self.w + torch.mul(self.alpha, hebb))))
        # print(input,input.shape)
        # print(yout, yout.shape)
        # print("Hebb:")
        # print(hebb)
        return yout, hebb

    def initialZeroState(self):
        return Variable(torch.zeros(1, self.params['nbneur']).type(ttype))

    def initialZeroHebb(self):
        return Variable(torch.zeros(self.params['nbneur'], self.params['nbneur']).type(ttype))

def train(paramdict=None):
    # params = dict(click.get_current_context().params)
    print("Starting training...")
    params = {}
    params.update(defaultParams)
    if paramdict:
        params.update(paramdict)
    print("Passed params: ", params)
    print(platform.uname())
    sys.stdout.flush()
    params['nbsteps'] = params['nbprescycles'] * (
                (params['prestime'] + params['interpresdelay']) * params['nbpatterns']) + params[
                            'prestimetest']  # Total number of steps per episode
    params['nbneur'] = params['patternsize'] + 1
    suffix = "images_" + "".join([str(x) + "_" if pair[0] is not 'nbneur' and pair[0] is not 'nbsteps' and pair[
        0] is not 'print_every' and pair[0] is not 'rngseed' else '' for pair in zip(params.keys(), params.values()) for
                                  x in pair])[:-1] + '_rngseed_' + str(
        params['rngseed'])  # Turning the parameters into a nice suffix for filenames; rngseed always appears last

    # Initialize random seeds (first two redundant?)
    print("Setting random seeds")
    np.random.seed(params['rngseed']);
    random.seed(params['rngseed']);
    torch.manual_seed(params['rngseed'])
    # print(click.get_current_context().params)

    print("Initializing network")
    net = Network(params)
    total_loss = 0.0

    print("Initializing optimizer")
    optimizer = torch.optim.Adam([net.w, net.alpha, net.eta], lr=params['lr'])
    all_losses = []
    # print_every = 20
    nowtime = time.time()
    print("Starting episodes...")
    sys.stdout.flush()

    yout=[]
    for numiter in range(params['nbiter']):
        # print("Iter ", numiter)
        # sys.stdout.flush()
        y = net.initialZeroState()
        hebb = net.initialZeroHebb()
        optimizer.zero_grad()

        # inputs, target = generateTestInput(params)
        inputs, target = generateTrainArr(params, y, Tout=False)

        # Running the episode
        for numstep in range(params['nbsteps']):
            y, hebb = net(Variable(inputs[numstep], requires_grad=False), y, hebb)

        # Computing gradients, applying optimizer
        loss = (y[0][:params['patternsize']] - Variable(target, requires_grad=False)).pow(2).sum()
        loss.backward()
        optimizer.step()

        yout = y

        # lossnum = loss.data[0]
        lossnum = loss.item()
        total_loss += lossnum

        # Printing statistics, saving files
        if (numiter + 1) % params['print_every'] == 0:

            print(numiter, "====")
            td = target.cpu().numpy()
            yd = y.data.cpu().numpy()[0][:-1]
            print("y: ", yd[:10])
            print("target: ", td[:10])

            absdiff = np.abs(td - yd)
            print("Mean / median / max abs diff:", np.mean(absdiff), np.median(absdiff), np.max(absdiff))
            print("Correlation (full / sign): ", np.corrcoef(td, yd)[0][1], np.corrcoef(np.sign(td), np.sign(yd))[0][1])

            previoustime = nowtime
            nowtime = time.time()
            print("Time spent on last", params['print_every'], "iters: ", nowtime - previoustime)
            total_loss /= params['print_every']
            all_losses.append(total_loss)
            print("Mean loss over last", params['print_every'], "iters:", total_loss)
            print("Saving local files...")
            sys.stdout.flush()
            with open('results_' + suffix + '.dat', 'wb') as fo:
                pickle.dump(net.w.data.cpu().numpy(), fo)
                pickle.dump(net.alpha.data.cpu().numpy(), fo)
                pickle.dump(net.eta.data.cpu().numpy(), fo)
                pickle.dump(all_losses, fo)
                pickle.dump(params, fo)
            print("ETA:", net.eta.data.cpu().numpy())
            with open('loss_' + suffix + '.txt', 'w') as thefile:
                for item in all_losses:
                    thefile.write("%s\n" % item)

            sys.stdout.flush()
            sys.stderr.flush()

            total_loss = 0
        print("yout:",yout)

        for nc in range(params['nbprescycles']):  # 3
        # np.random.shuffle(patterns)
            for nn in range(params['prestime']):  # 20
                numi = nc * (params['nbpatterns'] * (params['prestime'])) + 2 * (  #2 ii
                        params['prestime']) + nn
                inputs[numi][0][:params['patternsize']] = yout

@click.command()
@click.option('--nbpatterns', default=defaultParams['nbpatterns'])
@click.option('--nbprescycles', default=defaultParams['nbprescycles'])
@click.option('--homogenous', default=defaultParams['prestime'])
@click.option('--prestime', default=defaultParams['prestime'])
@click.option('--prestimetest', default=defaultParams['prestimetest'])
@click.option('--interpresdelay', default=defaultParams['interpresdelay'])
@click.option('--patternsize', default=defaultParams['patternsize'])
@click.option('--nbiter', default=defaultParams['nbiter'])
@click.option('--probadegrade', default=defaultParams['probadegrade'])
@click.option('--lr', default=defaultParams['lr'])
@click.option('--print_every', default=defaultParams['print_every'])
@click.option('--rngseed', default=defaultParams['rngseed'])
def main(nbpatterns, nbprescycles, homogenous, prestime, prestimetest, interpresdelay, patternsize, nbiter,
         probadegrade, lr, print_every, rngseed):
    train(paramdict=dict(click.get_current_context().params))
    # print(dict(click.get_current_context().params))

    # writer = SummaryWriter('./Result')
    model_m = Network(params)
    # dum_input = np.zeros((1, params['nbneur']))
    dum_imput = Variable(inputs[numstep], requires_grad=False)

    dum_in = model_m.initialZeroState()
    dum_hebb = net.initialZeroHebb()
    dum_y, dum_hebb = net(dum_input, dum_y, dum_hebb)

    with SummaryWriter(comment='PlasNetwork') as W:
        W.add_graph(net, (dum_in, y, hebb), verbose=True)

    with SummaryWriter(comment='PlasNetwork') as W:
        W.add_graph(model_m, dum_input, dum_in, dum_hebb)

if __name__ == "__main__":
    # train()
    main()


import os
import sys
import csv
import pandas as pd
from PIL import Image
from PIL import ImageStat

# modify root_dir according to your repo
root_dir = '/home/stuartsul/J6-Sampyo_SediNet'

gravel_dir_temp = root_dir+'/temp_images/gravel'
sand_dir_temp = root_dir+'/temp_images/sand'
image_dir = root_dir+'/images'

#brightness threshold
thr = 20
#brightness function
def brightness (im_file):
    try:
        im = im_file.convert('L')
        stat = ImageStat.Stat(im)
        return stat.mean[0]
    except IOError:
        print(im_file + "IOError")
        return 0

# show status data
total_num = 0
processed_num = 0
deleted_num = 0

if(sys.argv[1] == "gravel") or (sys.argv[1] == "both"):
    gravel_image_list = os.listdir(gravel_dir_temp)

    gravel_rawdata = pd.read_csv(root_dir+"/data/gravel-dist-rawdata.csv", encoding='utf-8')
    fg_csv = open(root_dir+'/data/gravel.csv','a')
    g_csv = csv.writer(fg_csv)

    for image in gravel_image_list:
        print("image: " + image)
        total_num = total_num + 1

        # rename image : "RAW-OS-GXXX-XX.jpg" -> "BIG-OS-GXXX-XX.jpg"
        NEW = image.split('-')[0]
        if(NEW != "RAW"):
            continue
        LOC = image.split('-')[1]
        NAME = image.split('-')[2]
        NUM = image.split('-')[3]
        new_name = '/BIG-'+LOC+'-'+NAME+'-'+NUM

        # delete tilted image
        NUMBER = NUM.split('.')[0]
        if(NUMBER.isdigit()):
            if(int(NUMBER)%8>=5 or int(NUMBER)%8==0):
                deleted_num = deleted_num + 1
                print(str(deleted_num) + " : deleted due to tilted : " + image)
                os.remove(gravel_dir_temp+'/'+image)
                continue

        # write csv files for images
        temp_line = gravel_rawdata.loc[gravel_rawdata['fac'].str.contains(LOC, na=False)]
        data_line = temp_line.loc[temp_line['no'].str.contains(NAME, na=False)]

        # if no data is in gravel-dist-rawdata.csv
        if (data_line.empty):
            deleted_num = deleted_num + 1
            print(str(deleted_num) + " : deleted due to no data : " + image)
            os.remove(gravel_dir_temp+'/'+image)
            continue

        # open image and check its size
        # crop image and save it in image directory
        # delete RAW image
        try:
            img = Image.open(gravel_dir_temp+'/'+image)
            img_size = img.size
            if(img_size[0] <3500 or img_size[1] < 2350):
                deleted_num = deleted_num +1
                print(str(deleted_num) + " : deleted due to size : " + image)
                os.remove(gravel_dir_temp+'/'+image)
                continue

            # crop image
            area = (500, 650, 3500, 2350) # cropped area
            if(img_size[0] < img_size[1]):
                img_rotate = img.rotate(90,expand=True)
                crop_image = img_rotate.crop(area)
            else:
                crop_image = img.crop(area)

            # delete dark image
            br = brightness(crop_image)
            if(br <= thr):
                deleted_num = deleted_num + 1
                print(str(deleted_num) + " : deleted due to brightness : " + image)
            else:
                processed_num = processed_num + 1
                crop_image.save(image_dir+new_name)
                g_csv.writerow(['images'+new_name,data_line.iat[0,2],data_line.iat[0,3],data_line.iat[0,4],data_line.iat[0,5],data_line.iat[0,6],data_line.iat[0,7],data_line.iat[0,8],data_line.iat[0,9],data_line.iat[0,10],data_line.iat[0,11],data_line.iat[0,12]])

            # delete original RAW image in temp_images directory
            os.remove(gravel_dir_temp+'/'+image)
        except IOError:
            deleted_num = deleted_num + 1
            print(str(deleted_num) + " : deleted due to IOError : " + image)
            os.remove(gravel_dir_temp+'/'+image)
            continue

        if(total_num%100==0):
            print("total : " + str(total_num))
            print("processed : " +str(processed_num))
            print("deleted : " + str(deleted_num))

    fg_csv.close()