#!/usr/bin/env python3
# -*- coding: utf-8 -*-
import os
import sys
try:
  import petsclinter as pl
except ModuleNotFoundError as mnfe:
  try:
    petsc_dir = os.environ['PETSC_DIR']
  except KeyError as ke:
    raise RuntimeError('Must set PETSC_DIR environment variable') from ke
  sys.path.insert(0, os.path.join(petsc_dir, 'lib', 'petsc', 'bin', 'maint', 'petsclinter'))
  import petsclinter as pl

def __prepare_ns_args(ns_args, parser):
  slepc_mansecs     = ['eps','lme','mfn','nep','pep','svd','sys']
  slepc_aux_mansecs = ['bv','ds','fn','rg','st']

  if ns_args.slepc_dir is None:
    raise RuntimeError('Could not determine SLEPC_DIR from environment, please set via options')

  extra_compiler_flags = [
    '-I' + os.path.join(ns_args.slepc_dir, 'include'),
    '-I' + os.path.join(ns_args.slepc_dir, ns_args.petsc_arch, 'include')
  ]
  with open(os.path.join(ns_args.slepc_dir, ns_args.petsc_arch, 'lib', 'slepc', 'conf', 'slepcvariables'), 'r') as sv:
    line = sv.readline()
    while line:
      if 'INCLUDE' in line:
        for inc in line.split('=', 1)[1].split():
          extra_compiler_flags.append(inc)
      line = sv.readline()

  extra_header_includes = []
  mansecimpls           = [m + 'impl.h' for m in slepc_mansecs + slepc_aux_mansecs] + [
    'slepcimpl.h', 'vecimplslepc.h'
  ]
  for header_file in os.listdir(os.path.join(ns_args.slepc_dir, 'include', 'slepc', 'private')):
    if header_file in mansecimpls:
      extra_header_includes.append(f'#include <slepc/private/{header_file}>')

  if ns_args.src_path == parser.get_default('src_path'):
    ns_args.src_path = os.path.join(ns_args.slepc_dir, 'src')
  if ns_args.patch_dir == parser.get_default('patch_dir'):
    ns_args.patch_dir = os.path.join(ns_args.slepc_dir, 'slepcLintPatches')

  # prepend these
  ns_args.extra_compiler_flags = extra_compiler_flags + ns_args.extra_compiler_flags
  # replace these
  if not ns_args.extra_header_includes:
    ns_args.extra_header_includes = extra_header_includes

  return ns_args

def command_line_main():
  import argparse
  import petsclinter.main

  slepc_classid_map = {
    '_p_BV *'  : 'BV_CLASSID',
    '_p_DS *'  : 'DS_CLASSID',
    '_p_FN *'  : 'FN_CLASSID',
    '_p_RG *'  : 'RG_CLASSID',
    '_p_ST *'  : 'ST_CLASSID',
    '_p_EPS *' : 'EPS_CLASSID',
    '_p_PEP *' : 'PEP_CLASSID',
    '_p_NEP *' : 'NEP_CLASSID',
    '_p_SVD *' : 'SVD_CLASSID',
    '_p_MFN *' : 'MFN_CLASSID',
    '_p_LME *' : 'LME_CLASSID',
  }

  for struct_name, classid_name in slepc_classid_map.items():
    pl.checks.register_classid(struct_name, classid_name)

  parser      = argparse.ArgumentParser(prog='slepclinter', add_help=False)
  group_slepc = parser.add_argument_group(title='SLEPc location settings')
  group_slepc.add_argument('--SLEPC_DIR', required=False, default=os.environ.get('SLEPC_DIR', None), help='if this option is unused defaults to environment variable $SLEPC_DIR', dest='slepc_dir')

  args, parser = pl.main.parse_command_line_args(parent_parsers=[parser])
  args         = __prepare_ns_args(args, parser)

  return pl.main.namespace_main(args)

if __name__ == '__main__':
  sys.exit(command_line_main())


import cv2
import numpy as np
import csv
import math

img = cv2.imread(".//new_dataset//250.jpg")
img = cv2.GaussianBlur(img , (5 , 5) , 0)

n = 64
div = 256//n #n is the number of bins, here n = 64
rgb = cv2.split(img)
q = []
for ch in rgb:
    vf = np.vectorize(lambda x, div: int(x//div)*div)
    quantized = vf(ch, div)
    q.append(quantized.astype(np.uint8))
img = cv2.merge(q)

row , col , channels = img.shape

connectivity = 8
tau = 0
if tau == 0:
    tau = row*col*0.1

rgb = cv2.split(img)
q = []
for ch in rgb:
    vf = np.vectorize(lambda x, div: int(x//div)*div)
    quantized = vf(ch, div)
    q.append(quantized.astype(np.uint8))
img = cv2.merge(q)
bgr = cv2.split(img)
total = []
for ch in bgr:
    for k in range(0 , 256 , div):
        temp = ch.copy()
        temp = (temp == k).astype(np.uint8)
        output = cv2.connectedComponentsWithStats(temp , connectivity , cv2.CV_32S)
        num_labels = output[0]
        labels = output[0]
        stats = output[2]
        centroids = output[3]
        alpha = 0
        beta = 0
        req = stats[1:]
        for r in req:
            if(r[4] >= tau):
                alpha += r[4]
            else:
                beta += r[4]
        total.append(alpha)
        total.append(beta)

dist = []
name = []
with open('ccv_feat.csv' , 'r') as csvFile:
    reader = csv.reader(csvFile)
    i = 0
    for row in reader:
        distance = math.sqrt(sum([(float(a) - float(b)) ** 2 for a , b in zip(row , total)]))
        dist.append(distance)
        nam = str(i)+".jpg"
        name.append(nam)
        i += 1

di = {}
for i in range(len(name)):
    di[name[i]] = dist[i]
sorted_di = sorted(di.items() , key = lambda kv : kv[1])

pic = []
t = 0
first_key = list(sorted_di)[:10]
for key , val in first_key:
    key = ".//new_dataset//"+key
    res = cv2.imread(key)
    na = str(t)
    cv2.imshow(na , res)
    t += 1


#!/usr/bin/env python3
# -*- coding: utf-8 -*-

from icalendar import Calendar, Event
from datetime import *
from dateutil.parser import parse
from pytz import UTC  # timezone

cal_file = '/Users/gyyoon/Desktop/pentaa.ics'

MEETING_STR = '[회의]'
TEATIME_STR = '[Tea-Time]'
WORK_STR = '[업무]'
NONWORK_STR = '[비업무]'
LUNCH_STR = '[점심]'
CLUB_STR = '[동호회]'
WORKCLEAN_STR = '[업무정리]'
TALK_STR = '[면담]'
OUTWORK_STR = '[외근]'
BREAK_STR = '[휴식]'
TOGETHER_CTR = '[회식]'
SEATCLEAN_STR = '[자리정리]'
EVENT_STR = '[행사]'
LATE_SRT = '[지연출근]'

# Count global var
total_cnt = 0
work_cnt = 0
nonwork_cnt = 0
meeting_cnt = 0
lunch_cnt = 0
club_cnt = 0
workclean_cnt = 0
outwork_cnt = 0
unknown_cnt = 0
break_cnt = 0
together_cnt = 0
seatclean_cnt = 0
event_cnt = 0

# Duration global var
work_dur = timedelta(hours=0, minutes=0)
nonwork_dur = timedelta(hours=0, minutes=0)
meeting_dur = timedelta(hours=0, minutes=0)
lunch_dur = timedelta(hours=0, minutes=0)
club_dur = timedelta(hours=0, minutes=0)
workclean_dur = timedelta(hours=0, minutes=0)
outwork_dur = timedelta(hours=0, minutes=0)
break_dur = timedelta(hours=0, minutes=0)
together_dur = timedelta(hours=0, minutes=0)
seatclean_dur = timedelta(hours=0, minutes=0)
event_dur = timedelta(hours=0, minutes=0)
late_dur = timedelta(hours=0, minutes=0)

def process_work_event(comp):
    # Duration global var
    global work_dur

    start_dt = comp.get('dtstart').dt
    end_dt = comp.get('dtend').dt
    duration = end_dt - start_dt

    work_dur += duration

def process_nonwork_event(comp):
    # Duration global var
    global nonwork_dur

    start_dt = comp.get('dtstart').dt
    end_dt = comp.get('dtend').dt
    duration = end_dt - start_dt

    nonwork_dur += duration

def process_meeting_event(comp):
    # Duration global var
    global meeting_dur

    start_dt = comp.get('dtstart').dt
    end_dt = comp.get('dtend').dt
    duration = end_dt - start_dt

    meeting_dur += duration

def process_lunch_event(comp):
    # Duration global var
    global lunch_dur

    start_dt = comp.get('dtstart').dt
    end_dt = comp.get('dtend').dt
    duration = end_dt - start_dt

    lunch_dur += duration

def process_club_event(comp):
    # Duration global var
    global club_dur

    start_dt = comp.get('dtstart').dt
    end_dt = comp.get('dtend').dt
    duration = end_dt - start_dt

    club_dur += duration

def process_workclean_event(comp):
    # Duration global var
    global workclean_dur

    start_dt = comp.get('dtstart').dt
    end_dt = comp.get('dtend').dt
    duration = end_dt - start_dt

    workclean_dur += duration

def process_outwork_event(comp):
    # Duration global var
    global outwork_dur

    start_dt = comp.get('dtstart').dt
    end_dt = comp.get('dtend').dt
    duration = end_dt - start_dt

    outwork_dur += duration

def process_break_event(comp):
    # Duration global var
    global break_dur

    start_dt = comp.get('dtstart').dt
    end_dt = comp.get('dtend').dt
    duration = end_dt - start_dt

    break_dur += duration

def process_together_event(comp):
    # Duration global var
    global together_dur

    start_dt = comp.get('dtstart').dt
    end_dt = comp.get('dtend').dt
    duration = end_dt - start_dt

    together_dur += duration

def process_seatclean_event(comp):
    # Duration global var
    global seatclean_dur

    start_dt = comp.get('dtstart').dt
    end_dt = comp.get('dtend').dt
    duration = end_dt - start_dt

    seatclean_dur += duration

def process_event_event(comp):
    # Duration global var
    global event_dur

    start_dt = comp.get('dtstart').dt
    end_dt = comp.get('dtend').dt
    duration = end_dt - start_dt

    event_dur += duration

def process_late_event(comp):
    global late_dur

    start_dt = comp.get('dtstart').dt
    end_dt = comp.get('dtend').dt
    duration = end_dt - start_dt

    late_dur += duration

def print_results():
    print('----------------------------------')

    print('total count : ' + str(total_cnt))
    print('meeting : ' + str(meeting_cnt))
    print('work : ' + str(work_cnt))
    print('nonwork : ' + str(nonwork_cnt))
    print('lunch : ' + str(lunch_cnt))
    print('unknown ' + str(unknown_cnt))

    print('----------------------------------')

    print('workdur : ' + str(work_dur))
    print('nonwork dur : ' + str(nonwork_dur))
    print('meeting dur : ' + str(meeting_dur))
    print('lunch dur : ' + str(lunch_dur))
    print('club dur : ' + str(club_dur))
    print('workclean dur : ' + str(workclean_dur))
    print('outwork dur : ' + str(outwork_dur))
    print('break dur : ' + str(break_dur))
    print('together dur : ' + str(together_dur))
    print('seatclean dur : ' + str(seatclean_dur))
    print('event dur : ' + str(event_dur))
    print('late dur : ' + str(late_dur))

    print('----------------------------------')

class EventProcessor:
    def __init__(self):
        print('Initialize')

    def calc_event_duration(self, comp):
        start_dt = comp.get('dtstart').dt
        end_dt = comp.get('dtend').dt
        duration = end_dt - start_dt

        return duration

def main():
    # Count global var
    global total_cnt
    global work_cnt
    global nonwork_cnt
    global meeting_cnt
    global lunch_cnt
    global club_cnt
    global workclean_cnt
    global outwork_cnt
    global break_cnt
    global together_cnt
    global seatclean_cnt
    global event_cnt
    global unknown_cnt
    global late_dur

    g = open(cal_file, 'rb')
    gcal = Calendar.from_ical(g.read())

    for component in gcal.walk():
        if component.name == "VEVENT":
            start_date = component.get('dtstart').dt
            end_date = component.get('dtend').dt

            # I don't know why, but same formatted strs have differecne types,
            # date or datetime, so I unified them to date type(datetime requires
            # timezone set...)
            if type(start_date) != type(date(2018, 4, 4)):
                start_date = start_date.date()

            if type(end_date) != type(date(2018, 4, 4)):
                end_date = end_date.date()

            # Maybe someday I might get inputs from user that specifies
            # the date range...
            if start_date >= date(2018, 1, 1) and end_date <= date(2018, 12, 31):
                total_cnt += 1

                event_summary = component.get('summary')

                if WORK_STR in event_summary:
                    work_cnt += 1
                    process_work_event(component)
                    print(event_summary)

                elif MEETING_STR in event_summary or TEATIME_STR in \
                        event_summary or TALK_STR in event_summary:
                    meeting_cnt += 1
                    process_meeting_event(component)

                elif NONWORK_STR in event_summary:
                    nonwork_cnt += 1
                    process_nonwork_event(component)

                elif LUNCH_STR in event_summary:
                    lunch_cnt += 1
                    process_lunch_event(component)

                elif CLUB_STR in event_summary:
                    club_cnt += 1
                    process_club_event(component)

                elif WORKCLEAN_STR in event_summary:
                    workclean_cnt += 1
                    process_workclean_event(component)

                elif OUTWORK_STR in event_summary:
                    outwork_cnt += 1
                    process_outwork_event(component)

                elif BREAK_STR in event_summary:
                    break_cnt += 1
                    process_break_event(component)

                elif TOGETHER_CTR in event_summary:
                    together_cnt += 1
                    process_together_event(component)

                elif SEATCLEAN_STR in event_summary:
                    seatclean_cnt += 1
                    process_seatclean_event(component)

                elif EVENT_STR in event_summary:
                    event_cnt += 1
                    process_event_event(component)
                elif LATE_SRT in event_summary:
                    event_cnt += 1
                    process_late_event(component)
                else:
                    unknown_cnt += 1
                    # print(event_summary)
                    # print(component.get('dtstart').dt)
                    # print(component.get('dtend').dt)
                    # print(component.get('dtstamp').dt)
    g.close()

    print_results()

if __name__ == "__main__":
    main()


import code

import gym
import torch
from tqdm import trange

import numpy as np

import components.prioritized_memory
import components.memory
from components.filesys_manager import ExperimentPath

class BaseTrainer:

    def __init__(self, args):
        # init experiment hyper-parameters
        self.args = args
        self.debug = self.args.debug
        self.allow_printing = True
        self.exp_path = ExperimentPath(f"{self.args.exp_root}/{self.args.name}/{self.args.env}/{args.trainer}/{self.args.seed}")
        self.print("log directory initialized")
        # env init
        self.expl_env = gym.make(self.args.env)
        self.eval_env = gym.make(self.args.env)
        # replay buffer
        if args.prioritized:
            self.replay_buffer = components.prioritized_memory.ReplayMemory(self.args)
        else:
            self.replay_buffer = components.memory.ReplayMemory(self.args)
        # global time steps
        self.t = 0
        # exploration environment
        self.s = torch.from_numpy(self.expl_env.reset()).to(dtype=torch.float32, device=self.args.device)
        self.done = False
        self.episode_num = 0
        self.episode_len = 0
        self.episode_return = 0
        self.episode_discounted_return = 0
        # init model
        self.init_model()
        # log
        self.exp_path['timestamp[timenow,t,msg]'].csv_writerow([self.exp_path.now(), self.t, "model initialized"])
        self.exp_path['config'].json_write(vars(self.args))
        self.exp_path['model_info'].txt_write(str(self.q_learner))
        self.save('initial')

    def print(self, *args, **kwargs):
        if self.allow_printing:
            print(",", self.exp_path.now(), self.exp_path.str(), ",", *args, **kwargs)

    def init_model(self):
        """ init model """
        self.print("warning: init model not implemented")
        self.q_learner = None
        self.q_target = None
        self.policy = None
        self.q_optimizer = None
        self.policy_optimizer = None

    def sample_batch(self, batch_size):
        return self.replay_buffer.sample(batch_size)

    def expl_action(self, obs) -> torch.Tensor:
        self.print("warning: expl action not implemented")
        a = np.random.uniform(low=-1, high=1, size=self.expl_env.action_space.shape[0])
        return torch.from_numpy(a)

    def eval_action(self, obs) -> torch.Tensor:
        self.print("warning: eval action not implemented")
        a = np.random.uniform(low=-1, high=1, size=self.expl_env.action_space.shape[0])
        return torch.from_numpy(a)

    def before_learn_on_batch(self):
        self.print("warning: before learn on batch not implemented")

    def learn_on_batch(self, treeidx, s, a, r, s_, d, w):
        """ learn on batch """
        self.print("warning: learn on batch not implemented")

    def after_learn_on_batch(self):
        self.print("warning: after learn on batch not implemented")

    def state_value_pred(self, s):
        """ log the state value of initial state """
        self.print("warning: state value pred not implemented")
        return -1

    def reset_expl_env(self):
        """ reset exploration env """
        self.s = self.expl_env.reset()
        self.s = torch.from_numpy(self.s).to(dtype=torch.float32, device=self.args.device)
        self.done = False
        self.episode_num += 1
        self.episode_len = 0
        self.episode_return = 0
        self.episode_discounted_return = 0

    def advance_expl_env(self, next_obs, reward, done):
        """ advance exploration env for next step """
        self.s = next_obs
        self.done = done
        self.episode_return += reward
        if self.args.reward_clip:
            reward = min(max(reward, -1), 1)
        self.episode_discounted_return += reward * self.args.discount ** self.episode_len
        self.episode_len += 1

    def epsilon(self):
        """ compute current epsilon (linear annealing 1M steps) """
        return 1 - self.t * ((1 - 0.1) / self.args.epsilon_steps) if self.t < self.args.epsilon_steps else 0.1

    def collect_init_steps(self, render=False):
        """ fill the replay buffer before learning """
        for _ in trange(self.args.min_num_steps_before_training):
            if self.done:
                self.reset_expl_env()
            # sample transition
            a = torch.from_numpy(np.random.uniform(low=-1, high=1, size=self.expl_env.action_space.shape[0]))
            s_, r, d, _ = self.expl_env.step(a)
            s_ = torch.from_numpy(s_).to(dtype=torch.float32, device=self.args.device)
            r = r.item()
            # store into replay
            if self.args.reward_clip:
                r = min(max(r, -1), 1)
            if 'reward_std' in self.args:
                r += np.random.normal(0, self.args.reward_std)  # clip, add noise
            if self.args.prioritized:
                self.replay_buffer.append(self.s, a, r, d)
            else:
                self.replay_buffer.append(self.s, a, r, s_, d)
            # render
            if render:
                self.expl_env.render()
            # sample next state
            self.advance_expl_env(s_, r, d)
            self.t += 1
        self.reset_expl_env()
        self.print("finished collect init steps")

    def sample_transition(self, render=False):
        """ train one step """
        if self.done:
            self.reset_expl_env()

        # sample transition
        a = self.expl_action(self.s)
        s_, r, d, _ = self.expl_env.step(a)
        s_ = torch.from_numpy(s_).to(dtype=torch.float32, device=self.args.device)
        r = r.item()
        # store into replay
        if self.args.reward_clip:
            r = min(max(r, -1), 1)
        if 'reward_std' in self.args:
           r += np.random.normal(0, self.args.reward_std)  # clip, add noise
        if self.args.prioritized:
            self.replay_buffer.append(self.s, a, r, d)
        else:
            self.replay_buffer.append(self.s, a, r, s_, d)
        # render
        if render:
            self.expl_env.render()
        # sample next state
        self.advance_expl_env(s_, r, d)

        # log status
        if self.done:
            self.exp_path['expl_episode_stats[t,return,discounted_return]'].csv_writerow(
                [self.t, self.episode_return, self.episode_discounted_return])
            if self.args.prioritized:
                self.exp_path['debug[t,buffer_size,epsilon]'].csv_writerow(
                    [self.t, self.replay_buffer.data_buffer.index, self.epsilon()]
                )
            else:
                self.exp_path['debug[t,buffer_size]'].csv_writerow(
                    [self.t, len(self.replay_buffer)]
                )

        self.t += 1

    def evaluate(self, render=False, random=False):
        """ evaluate current policy """
        self.exp_path['timestamp[timenow,t,msg]'].csv_writerow([self.exp_path.now(), self.t, "evaluation started"])
        self.print(f"evaluation started t={self.t}")
        # init eval
        s = self.eval_env.reset()
        s = torch.from_numpy(s).to(dtype=torch.float32, device=self.args.device)
        done = False
        episode_len = 0
        episode_return = 0
        episode_discounted_return = 0
        v_initials = []
        v_initials.append(self.state_value_pred(s))
        returns = []
        discounted_returns = []
        episode_lens = []
        # start eval
        for _ in trange(self.args.num_eval_steps_per_epoch):
            if done:
                # record data
                returns.append(episode_return)
                discounted_returns.append(episode_discounted_return)
                episode_lens.append(episode_len)
                # reset env
                s = self.eval_env.reset()
                s = torch.from_numpy(s).to(dtype=torch.float32, device=self.args.device)
                v_initials.append(self.state_value_pred(s))
                episode_len = 0
                episode_return = 0
                episode_discounted_return = 0
            if len(returns) >= self.args.eval_max_episode:
                # check if enough episodes simulated
                break
            # sample transition
            a = self.eval_action(s)
            if random:
                a = np.random.randint(self.eval_env.action_space())
            s_, r, d, _ = self.eval_env.step(a)
            s_ = torch.from_numpy(s_).to(dtype=torch.float32, device=self.args.device)
            r = r.item()
            # advance env
            s = s_
            done = d
            episode_return += r
            if self.args.reward_clip:
                r = min(max(r, -1), 1)
            episode_discounted_return += r * self.args.discount ** episode_len
            episode_len += 1
            # render
            if render:
                print(f"s={s.shape} a={a} r={r} r_clip={r}")
                self.eval_env.render()
        # finished eval
        self.print(f"returns={returns}\nmean={np.mean(returns) if len(returns) > 0 else 0}")
        if len(returns) == 0:
            self.exp_path['eval_episode_stats[t,v_init,return,discounted_return,length]'].csv_writerow(
                [self.t, v_initials[0], episode_return, episode_discounted_return, episode_len])
            self.exp_path['eval_mean_stats[t,num_episode,v_init,return,discounted_return,length]'].csv_writerow(
                [self.t, 1, v_initials[0], episode_return, episode_discounted_return, episode_len])

        else:
            for V_init, R, discounted_R, length in zip(v_initials, returns, discounted_returns, episode_lens):
                self.exp_path['eval_episode_stats[t,v_init,return,discounted_return,length]'].csv_writerow(
                    [self.t, V_init, R, discounted_R, length])
            self.exp_path['eval_mean_stats[t,num_episode,v_init,return,discounted_return,length]'].csv_writerow(
                [self.t, len(returns), np.mean(v_initials), np.mean(returns), np.mean(discounted_returns), np.mean(episode_lens)])

        self.exp_path['timestamp[timenow,t,msg]'].csv_writerow([self.exp_path.now(), self.t, "evaluation finished"])

    def save(self, name='trainer'):
        """ save this trainer """
        buffer = self.replay_buffer
        expl_env = self.expl_env
        eval_env = self.eval_env
        # skip the things not saving
        self.replay_buffer = None
        self.expl_env = None
        self.eval_env = None
        self.exp_path["checkpoint"][f"{name}_{self.t}.pth"].save_model(self)
        self.exp_path['timestamp[timenow,t,msg]'].csv_writerow([self.exp_path.now(), self.t, "model saved"])
        self.replay_buffer = buffer
        self.expl_env = expl_env
        self.eval_env = eval_env


import cv2
import math

# Source: https://richardpricejones.medium.com/drawing-a-rectangle-with-a-angle-using-opencv-c9284eae3380
# Made slight adjustments to color
def draw_angled_rec(x0, y0, width, height, angle, img, color):

    _angle = angle * math.pi / 180.0
    b = math.cos(_angle) * 0.5
    a = math.sin(_angle) * 0.5
    pt0 = (int(x0 - a * height - b * width),
           int(y0 + b * height - a * width))
    pt1 = (int(x0 + a * height - b * width),
           int(y0 - b * height - a * width))
    pt2 = (int(2 * x0 - pt0[0]), int(2 * y0 - pt0[1]))
    pt3 = (int(2 * x0 - pt1[0]), int(2 * y0 - pt1[1]))

    if color == 'green':
        cv2.line(img, pt0, pt1, (0, 255, 0), 5)
        cv2.line(img, pt1, pt2, (0, 255, 0), 5)
        cv2.line(img, pt2, pt3, (0, 255, 0), 5)
        cv2.line(img, pt3, pt0, (0, 255, 0), 5)
    elif color == 'red':
        cv2.line(img, pt0, pt1, (0, 0, 255), 5)
        cv2.line(img, pt1, pt2, (0, 0, 255), 5)
        cv2.line(img, pt2, pt3, (0, 0, 255), 5)
        cv2.line(img, pt3, pt0, (0, 0, 255), 5)
    else:
        cv2.line(img, pt0, pt1, (255, 0, 255), 5)
        cv2.line(img, pt1, pt2, (255, 0, 255), 5)
        cv2.line(img, pt2, pt3, (255, 0, 255), 5)
        cv2.line(img, pt3, pt0, (255, 0, 255), 5)

img = cv2.imread('minAreaRect_Test.png')
img_gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
ret, thresh = cv2.threshold(img_gray, 150, 255, cv2.THRESH_BINARY)
# detect the contours on the binary image using cv2.CHAIN_APPROX_NONE
contours, hierarchy = cv2.findContours(image=thresh, mode=cv2.RETR_TREE, method=cv2.CHAIN_APPROX_NONE)
slender_rat = 3
min_width = 10
max_width = 120
min_len = 120
first = 1
image_copy = img.copy()
cv2.imwrite('minAreaRect_Test_Result.png',image_copy)
# cv2.drawContours(image=image_copy, contours=contours, contourIdx=-1, color=(0, 255, 0), thickness=2, lineType=cv2.LINE_AA)
# Used to add text on pattern
counter = 0
for cnt in contours:
    if counter == 0:        # First contour encompasses entire image
        counter += 1
        continue

    heir = hierarchy[0][counter][3]                         # [next, previous, first child, parent].
    if heir == 0:
        rect = cv2.minAreaRect(cnt)
        x = int(rect[0][0])
        y = int(rect[0][1])
        w = int(rect[1][0])
        h = int(rect[1][1])
        theta = int(rect[2])
        draw_angled_rec(x, y, w, h, theta, image_copy, 'green')
        image_tmp = cv2.putText(img=image_copy, text=str(theta)+'[deg]', org=(x, y), fontFace=cv2.FONT_HERSHEY_SIMPLEX, fontScale=3, color=(0,0,0), thickness=5)
        image_tmp = cv2.putText(img=image_copy, text='w='+str(w), org=(x, y+100), fontFace=cv2.FONT_HERSHEY_SIMPLEX, fontScale=3, color=(0,0,0), thickness=5)
        image_tmp = cv2.putText(img=image_copy, text='h='+str(h), org=(x, y+200), fontFace=cv2.FONT_HERSHEY_SIMPLEX, fontScale=3, color=(0,0,0), thickness=5)
        cv2.imwrite('minAreaRect_Test_Result.png',image_copy)
    counter += 1


__docformat__ = "reStructuredText"
__author__ = "davidh"

import logging
import os
import re
import sys
from collections import namedtuple
from datetime import datetime, timedelta
from glob import glob

LOG = logging.getLogger(__name__)

FILENAME_RE = r"HS_H08_(?P<date>\d{8})_(?P<time>\d{4})_(?P<band>B\d{2})_FLDK_(?P<res>R\d+)\.(?P<ext>.+)"
fn_re = re.compile(FILENAME_RE)

DT_FORMAT = "%Y%m%d_%H%M"
CASE_NAME_FORMAT = "{start}_{end}_{delta:02d}"

DataCase = namedtuple("DataCase", ["topic_title", "start", "end", "delta", "bands"])

### Guam Cases ###
guam_cases: dict = {}
# Kathy's Cases
guam_cases["Introduction"] = []
guam_cases["Introduction"].append(
    DataCase(
        "Introduction", datetime(2015, 7, 17, 21, 0, 0), datetime(2015, 7, 18, 20, 0, 0), timedelta(minutes=60), "all"
    )
)
guam_cases["Introduction"].append(
    DataCase(
        "Introduction", datetime(2015, 7, 18, 1, 0, 0), datetime(2015, 7, 18, 3, 20, 0), timedelta(minutes=10), "all"
    )
)
guam_cases["Introduction"].append(
    DataCase(
        "Introduction", datetime(2015, 7, 18, 14, 0, 0), datetime(2015, 7, 18, 16, 0, 0), timedelta(minutes=10), "all"
    )
)
guam_cases["Introduction"].append(
    DataCase("Introduction", datetime(2016, 3, 9, 0, 0, 0), datetime(2016, 3, 9, 4, 0, 0), timedelta(minutes=60), "all")
)
guam_cases["Introduction"].append(
    DataCase(
        "Introduction", datetime(2016, 3, 9, 1, 30, 0), datetime(2016, 3, 9, 4, 0, 0), timedelta(minutes=10), "all"
    )
)

# Scott's Cases
guam_cases["Water Vapor"] = []
guam_cases["Water Vapor"].append(
    DataCase(
        "Water Vapor", datetime(2015, 10, 7, 0, 0, 0), datetime(2015, 10, 8, 0, 0, 0), timedelta(minutes=30), "all"
    )
)
guam_cases["Water Vapor"].append(
    DataCase(
        "Water Vapor", datetime(2016, 2, 19, 19, 0, 0), datetime(2016, 2, 20, 5, 0, 0), timedelta(minutes=60), "all"
    )
)

# Tim's Cases
guam_cases["Weighting Functions"] = []
guam_cases["Weighting Functions"].append(
    DataCase(
        "Weighting Functions",
        datetime(2015, 9, 20, 2, 30, 0),
        datetime(2015, 9, 20, 2, 30, 0),
        timedelta(minutes=0),
        "all",
    )
)
guam_cases["Weighting Functions"].append(
    DataCase(
        "Weighting Functions",
        datetime(2015, 9, 20, 0, 0, 0),
        datetime(2015, 9, 20, 6, 0, 0),
        timedelta(minutes=60),
        "all",
    )
)
guam_cases["Weighting Functions"].append(
    DataCase(
        "Weighting Functions",
        datetime(2015, 9, 20, 1, 30, 0),
        datetime(2015, 9, 20, 2, 30, 0),
        timedelta(minutes=10),
        "all",
    )
)
guam_cases["Weighting Functions"].append(
    DataCase(
        "Weighting Functions",
        datetime(2015, 9, 20, 1, 0, 0),
        datetime(2015, 9, 20, 3, 0, 0),
        timedelta(minutes=10),
        "all",
    )
)

# Jordan's Cases
guam_cases["Extra"] = []
guam_cases["Extra"].append(
    DataCase("Extra", datetime(2015, 8, 17, 12, 0, 0), datetime(2015, 8, 18, 12, 0, 0), timedelta(minutes=60), "all")
)
guam_cases["Extra"].append(
    DataCase("Extra", datetime(2015, 8, 17, 22, 0, 0), datetime(2015, 8, 18, 1, 0, 0), timedelta(minutes=10), "all")
)
guam_cases["Extra"].append(
    DataCase("Extra", datetime(2015, 8, 24, 15, 0, 0), datetime(2015, 8, 15, 21, 0, 0), timedelta(minutes=60), "all")
)
guam_cases["Extra"].append(
    DataCase("Extra", datetime(2015, 8, 25, 2, 0, 0), datetime(2015, 8, 25, 5, 0, 0), timedelta(minutes=10), "all")
)

def main():
    import argparse

    parser = argparse.ArgumentParser(description="Regenerate or generate mirrored AHI data structure")
    parser.add_argument(
        "base_ahi_dir",
        default="/odyssey/isis/tmp/davidh/sift_data/ahi",
        help="Base AHI directory for the geotiff data files " "(next child directory is the full dated directory)",
    )
    parser.add_argument(
        "-v",
        "--verbose",
        dest="verbosity",
        action="count",
        default=int(os.environ.get("VERBOSITY", 2)),
        help="each occurrence increases verbosity 1 level through " "ERROR-WARNING-Info-DEBUG (default Info)",
    )
    parser.add_argument("--overwrite", action="store_true", help="Overwrite existing hardlinks")
    args = parser.parse_args()

    levels = [logging.ERROR, logging.WARN, logging.INFO, logging.DEBUG]
    level = levels[min(3, args.verbosity)]
    logging.basicConfig(level=level)

    if not os.path.isdir(args.base_ahi_dir):
        raise NotADirectoryError("Directory does not exist: %s" % (args.base_ahi_dir,))

    os.chdir(args.base_ahi_dir)

    for section_name, cases in guam_cases.items():
        for case in cases:
            start_str = case.start.strftime(DT_FORMAT)
            end_str = case.end.strftime(DT_FORMAT)
            # Note this only uses the minutes!
            case_name = CASE_NAME_FORMAT.format(
                start=start_str, end=end_str, delta=int(case.delta.total_seconds() / 60.0)
            )
            case_dir = os.path.join(args.base_ahi_dir, section_name, case_name)
            if not os.path.isdir(case_dir):
                LOG.debug("Creating case directory: %s", case_dir)
                os.makedirs(case_dir)
            else:
                LOG.error("Case directory already exists: %s", case_dir)
                continue

            t = case.start
            while t <= case.end:
                glob_pattern = t.strftime("%Y_%m_%d_%j/%H%M/*_%Y%m%d_%H%M_B??_*.merc.tif")
                t = t + case.delta

                matches = glob(glob_pattern)
                if len(matches) == 0:
                    LOG.error("Zero files found matching pattern: %s", glob_pattern)
                    continue
                for input_pathname in matches:
                    fn = os.path.basename(input_pathname)
                    link_path = os.path.join(case_dir, fn)
                    if os.path.exists(link_path) and not args.overwrite:
                        LOG.debug("Link '%s' already exists, skipping...", link_path)
                        continue
                    LOG.debug("Creating hardlink '%s' -> '%s'", link_path, input_pathname)
                    os.link(input_pathname, link_path)
                if int(case.delta.total_seconds()) == 0:
                    LOG.debug("Only one file needed to meet delta of 0")
                    break
            LOG.info("done mirroring files")

if __name__ == "__main__":
    sys.exit(main())


from django.db import models
from wagtail.admin.edit_handlers import MultiFieldPanel, RichTextFieldPanel, StreamFieldPanel
from wagtail.core.fields import RichTextField, StreamField
from wagtail.snippets.models import register_snippet

from ..modules import text_processing
from .. import configurations
from ..blogs.blocks import SectionBlock

@register_snippet
class BlogPost(models.Model):
    post_title = RichTextField(
        features=[], blank=False, null=True,
    )
    post_summary = RichTextField(
        features=configurations.RICHTEXT_FEATURES, blank=False, null=True,
    )
    post_introduction = RichTextField(
        features=configurations.RICHTEXT_FEATURES, blank=True, null=True,
    )
    post_conclusion = RichTextField(
        features=configurations.RICHTEXT_FEATURES, blank=True, null=True,
    )
    sections = StreamField(
        [
            ('section', SectionBlock()),
        ], blank=False
    )

    panels = [
        MultiFieldPanel(
            [
                RichTextFieldPanel('post_title'),
                RichTextFieldPanel('post_summary'),
                RichTextFieldPanel('post_introduction'),
                StreamFieldPanel('sections'),
                RichTextFieldPanel('post_conclusion'),
            ], heading='Post Content'
        ),
    ]

    @property
    def sections_with_title(self):
        sections = []
        for section in self.sections:
            if section.value['title']:
                sections.append(section)
        return sections

    def __str__(self):
        return text_processing.html_to_str(
            self.post_title
        )


#!/usr/bin/env python
# -*- coding: utf-8 -*-
#
# @Author: José Sánchez-Gallego (gallegoj@uw.edu)
# @Date: 2023-01-19
# @Filename: test_callback.py
# @License: BSD 3-clause (http://www.opensource.org/licenses/BSD-3-Clause)

import unittest.mock

import click
from click.testing import CliRunner

from unclick.core import build_command_string

@click.command()
@click.argument("ARG1", type=str)
@click.argument("ARG2", type=int, required=False)
@click.option("--flag1", "-f", is_flag=True, help="A flag.")
def my_command1(*args, **kwrgs):
    """Test command."""

    return

def _command_invoke(command: click.Command, string: str):
    """Checks that command is called and returns a mock of the callback."""

    with unittest.mock.patch.object(command, "callback") as mock_callback:
        runner = CliRunner()

        result = runner.invoke(command, string[len(command.name or "") :])
        assert result.exit_code == 0

        mock_callback.assert_called()

    return mock_callback

def test_callback():
    command_string = build_command_string(my_command1, "hi", 2, flag1=True)

    mock_callback = _command_invoke(my_command1, command_string)
    mock_callback.assert_called_once_with(arg1="hi", arg2=2, flag1=True)

def test_callback2():
    command_string = build_command_string(my_command1, "hi")

    mock_callback = _command_invoke(my_command1, command_string)
    mock_callback.assert_called_once_with(arg1="hi", arg2=None, flag1=False)

def test_callback_string_with_spaces():
    command_string = build_command_string(my_command1, "hi how are you")

    mock_callback = _command_invoke(my_command1, command_string)
    mock_callback.assert_called_once_with(arg1="hi how are you", arg2=None, flag1=False)


"""
Query builder examples.

NOTES:

# Infix notation (natural to humans)
    NOT ((FROM='11' OR TO="22" OR TEXT="33") AND CC="44" AND BCC="55")
# Prefix notation (Polish notation, IMAP version)
    NOT (((OR OR FROM "11" TO "22" TEXT "33") CC "44" BCC "55"))
# Python query builder
    NOT(AND(OR(from_='11', to='22', text='33'), cc='44', bcc='55'))

# python to prefix notation steps:
1. OR(1=11, 2=22, 3=33) ->
    "(OR OR FROM "11" TO "22" TEXT "33")"
2. AND("(OR OR FROM "11" TO "22" TEXT "33")", cc='44', bcc='55') ->
    "AND(OR(from_='11', to='22', text='33'), cc='44', bcc='55')"
3. NOT("AND(OR(from_='11', to='22', text='33'), cc='44', bcc='55')") ->
    "NOT (((OR OR FROM "1" TO "22" TEXT "33") CC "44" BCC "55"))"
"""

import datetime as dt
from imap_tools import AND, OR, NOT, A, H, U

# date in the date list (date=date1 OR date=date3 OR date=date2)
q1 = OR(date=[dt.date(2019, 10, 1), dt.date(2019, 10, 10), dt.date(2019, 10, 15)])
# '(OR OR ON 1-Oct-2019 ON 10-Oct-2019 ON 15-Oct-2019)'

# date not in the date list (NOT(date=date1 OR date=date3 OR date=date2))
q2 = NOT(OR(date=[dt.date(2019, 10, 1), dt.date(2019, 10, 10), dt.date(2019, 10, 15)]))
# 'NOT ((OR OR ON 1-Oct-2019 ON 10-Oct-2019 ON 15-Oct-2019))'

# subject contains "hello" AND date greater than or equal dt.date(2019, 10, 10)
q3 = A(subject='hello', date_gte=dt.date(2019, 10, 10))
# '(SUBJECT "hello" SINCE 10-Oct-2019)'

# from contains one of the address parts
q4 = OR(from_=["@spam.ru", "@tricky-spam.ru"])
# '(OR FROM "@spam.ru" FROM "@tricky-spam.ru")'

# marked as seen and not flagged
q5 = AND(seen=True, flagged=False)
# '(SEEN UNFLAGGED)'

# (text contains tag15 AND subject contains tag15) OR (text contains tag10 AND subject contains tag10)
q6 = OR(AND(text='tag15', subject='tag15'), AND(text='tag10', subject='tag10'))
# '(OR (TEXT "tag15" SUBJECT "tag15") (TEXT "tag10" SUBJECT "tag10"))'

# (text contains tag15 OR subject contains tag15) OR (text contains tag10 OR subject contains tag10)
q7 = OR(OR(text='tag15', subject='tag15'), OR(text='tag10', subject='tag10'))
# '(OR (OR TEXT "tag15" SUBJECT "tag15") (OR TEXT "tag10" SUBJECT "tag10"))'

# header IsSpam contains '++' AND header CheckAntivirus contains '-'
q8 = A(header=[H('IsSpam', '++'), H('CheckAntivirus', '-')])
# '(HEADER "IsSpam" "++" HEADER "CheckAntivirus" "-")'

# UID range
q9 = A(uid=U('1034', '*'))
# '(UID 1034:*)'

# complex from README
q10 = A(OR(from_='from@ya.ru', text='"the text"'), NOT(OR(A(answered=False), A(new=True))), to='to@ya.ru')
# '((OR FROM "from@ya.ru" TEXT "\\"the text\\"") NOT ((OR (UNANSWERED) (NEW))) TO "to@ya.ru")'


# -*- coding: utf-8 -*-
import os
import sys
import webbrowser

from invoke import task

docs_dir = 'docs'
build_dir = os.path.join(docs_dir, '_build')

@task
def readme(ctx, browse=False):
    ctx.run("rst2html.py README.rst > README.html")
    if browse:
        webbrowser.open_new_tab('README.html')

def build_docs(ctx, browse):
    ctx.run("sphinx-build %s %s" % (docs_dir, build_dir), echo=True)
    if browse:
        browse_docs(ctx)

@task
def clean_docs(ctx):
    ctx.run('rm -rf %s' % build_dir)

@task
def browse_docs(ctx):
    path = os.path.join(build_dir, 'index.html')
    webbrowser.open_new_tab(path)

@task
def docs(ctx, clean=False, browse=False, watch=False):
    """Build the docs."""
    if clean:
        clean_docs(ctx)
    if watch:
        watch_docs(ctx, browse=browse)
    else:
        build_docs(ctx, browse=browse)

@task
def watch_docs(ctx, browse=False, port=1234):
    """Run build the docs when a file changes."""
    try:
        import sphinx_autobuild  # noqa
    except ImportError:
        print('ERROR: watch task requires the sphinx_autobuild package.')
        print('Install it with:')
        print('    pip install sphinx-autobuild')
        sys.exit(1)
    ctx.run('sphinx-autobuild {0} --port={port} {1} {2}'.format(
        '--open-browser' if browse else '', docs_dir, build_dir, port=port),
            echo=True, pty=True)


'''
Algorithm: just count how many characters(frequency more than one regard it as one)
'''
#!/bin/python3

import sys
from collections import Counter
def stringConstruction(s):
    # Complete this function
    return len(Counter(s).values())

if __name__ == "__main__":
    q = int(input().strip())
    for a0 in range(q):
        s = input().strip()
        result = stringConstruction(s)
        print(result)


#!/usr/bin/env python3
# -*- coding: utf-8 -*-

"""
Written by Lucas Sinclair and Paul Rougieux

JRC biomass Project.
Unit D1 Bioeconomy.
"""

# Built-in modules #

# Third party modules #

# First party modules #
import autopaths
from autopaths            import Path
from autopaths.auto_paths import AutoPaths
from autopaths.tmp_path   import new_temp_dir
from plumbing.cache       import property_cached
from tqdm import tqdm

# Internal modules #
from cbmcfs3_runner.reports.scenario import ScenarioReport
from cbmcfs3_runner.pump.dataframes import concat_as_df

###############################################################################
class Scenario(object):
    """
    This object represents a modification of the input data for the purpose.
    A scenario can be harvest and economic scenario.
    Actual scenarios should inherit from this class.
    """

    all_paths = """
    /logs_summary.md
    """

    def __iter__(self): return iter(self.runners.values())
    def __len__(self):  return len(self.runners.values())

    def __getitem__(self, key):
        """Return a runner based on a country code."""
        return self.runners[key]

    def __init__(self, continent):
        # Save parent #
        self.continent = continent
        # This scenario dir #
        self.base_dir = Path(self.scenarios_dir + self.short_name + '/')
        # Automatically access paths based on a string of many subpaths #
        self.paths = AutoPaths(self.base_dir, self.all_paths)

    def __repr__(self):
        return '%s object with %i runners' % (self.__class__, len(self))

    def __call__(self, verbose=False):
        for code, steps in tqdm(self.runners.items()):
            for runner in steps:
                runner(interrupt_on_error=False, verbose=verbose)
        self.compile_log_tails()

    @property
    def runners(self):
        msg = "You should inherit from this class and implement this property."
        raise NotImplementedError(msg)

    @property
    def scenarios_dir(self):
        """Shortcut to the scenarios directory."""
        return self.continent.scenarios_dir

    @property_cached
    def report(self):
        return ScenarioReport(self)

    def compile_log_tails(self, step=-1):
        summary = self.paths.summary
        summary.open(mode='w')
        summary.handle.write("# Summary of all log file tails\n\n")
        summary.handle.writelines(r[step].tail for r in self.runners.values() if r[step])
        summary.close()

    # ------------------------------ Others ----------------------------------#
    def make_csv_zip(self, csv_name, dest_dir):
        """
        Will make a zip file will the specified CSV file from every country
        together and place it in the given destination directory.
        For instance you can do:

        >>> f = scenario.make_csv_zip('ipcc_pools', '~/exports/for_sarah/')
        >>> print(f)
        """
        # Files to put in the zip #
        files = {iso: rl[-1].post_processor.csv_maker.paths(csv_name)
                 for iso, rl in self.runners.items()}
        # Actual name of CSV file #
        csv_full_name = next(iter(files.items()))[1].name
        # Destination directory #
        dest_dir = Path(dest_dir)
        # If it's not a directory #
        assert isinstance(dest_dir, autopaths.dir_path.DirectoryPath)
        # Destination zip file #
        dest_zip = dest_dir + csv_full_name + '.zip'
        # Temporary directory #
        tmp_dir = new_temp_dir()
        zip_dir = tmp_dir + csv_full_name + '/'
        zip_dir.create()
        # Copy #
        for iso, f in files.items():
            try:
                f.copy(zip_dir + iso + '.csv')
            except Exception as e:
                print("no data in ", iso)
                print('Error loading data: '+ str(e))
        # Compress #
        zip_dir.zip_to(dest_zip)
        # Remove #
        tmp_dir.remove()
        # Return #
        return dest_zip

    def concat_as_df(self, *args, **kwargs):
        """A data frame with many countries together, crucial for analysis"""
        return concat_as_df(self, *args, **kwargs)


import asyncio
import logging
import os
import re
import warnings
from asyncio import Future
from functools import wraps
from inspect import signature
from typing import Any, Callable, Dict, Iterable, List, Optional, Tuple, Union

from tqdm.auto import tqdm

from rubrix._constants import (
    DATASET_NAME_REGEX_PATTERN,
    DEFAULT_API_KEY,
    RUBRIX_WORKSPACE_HEADER_NAME,
)
from rubrix.client.apis.datasets import Datasets
from rubrix.client.apis.metrics import MetricsAPI
from rubrix.client.apis.searches import Searches
from rubrix.client.datasets import (
    Dataset,
    DatasetForText2Text,
    DatasetForTextClassification,
    DatasetForTokenClassification,
)
from rubrix.client.metrics.models import MetricResults
from rubrix.client.models import (
    BulkResponse,
    Record,
    Text2TextRecord,
    TextClassificationRecord,
    TokenClassificationRecord,
)
from rubrix.client.sdk.client import AuthenticatedClient
from rubrix.client.sdk.commons.api import async_bulk
from rubrix.client.sdk.commons.errors import RubrixClientError
from rubrix.client.sdk.datasets import api as datasets_api
from rubrix.client.sdk.datasets.models import CopyDatasetRequest, TaskType
from rubrix.client.sdk.metrics import api as metrics_api
from rubrix.client.sdk.metrics.models import MetricInfo
from rubrix.client.sdk.text2text import api as text2text_api
from rubrix.client.sdk.text2text.models import (
    CreationText2TextRecord,
    Text2TextBulkData,
    Text2TextQuery,
)
from rubrix.client.sdk.text_classification import api as text_classification_api
from rubrix.client.sdk.text_classification.models import (
    CreationTextClassificationRecord,
    LabelingRule,
    LabelingRuleMetricsSummary,
    TextClassificationBulkData,
    TextClassificationQuery,
)
from rubrix.client.sdk.token_classification import api as token_classification_api
from rubrix.client.sdk.token_classification.models import (
    CreationTokenClassificationRecord,
    TokenClassificationBulkData,
    TokenClassificationQuery,
)
from rubrix.client.sdk.users import api as users_api
from rubrix.client.sdk.users.models import User
from rubrix.utils import setup_loop_in_thread

_LOGGER = logging.getLogger(__name__)

class _RubrixLogAgent:
    def __init__(self, api: "Api"):
        self.__api__ = api
        self.__loop__, self.__thread__ = setup_loop_in_thread()

    @staticmethod
    async def __log_internal__(api: "Api", *args, **kwargs):

        try:
            return await api.log_async(*args, **kwargs)
        except Exception as ex:
            _LOGGER.error(
                f"Cannot log data {args, kwargs}\n"
                f"Error of type {type(ex)}\n: {ex}. ({ex.args})"
            )
            raise ex

    def log(self, *args, **kwargs) -> Future:
        return asyncio.run_coroutine_threadsafe(
            self.__log_internal__(self.__api__, *args, **kwargs), self.__loop__
        )

class Api:
    # Larger sizes will trigger a warning
    _MAX_CHUNK_SIZE = 5000

    def __init__(
        self,
        api_url: Optional[str] = None,
        api_key: Optional[str] = None,
        workspace: Optional[str] = None,
        timeout: int = 60,
        extra_headers: Optional[Dict[str, str]] = None,
    ):
        """Init the Python client.

        We will automatically init a default client for you when calling other client methods.
        The arguments provided here will overwrite your corresponding environment variables.

        Args:
            api_url: Address of the REST API. If `None` (default) and the env variable ``RUBRIX_API_URL`` is not set,
                it will default to `http://localhost:6900`.
            api_key: Authentification key for the REST API. If `None` (default) and the env variable ``RUBRIX_API_KEY``
                is not set, it will default to `rubrix.apikey`.
            workspace: The workspace to which records will be logged/loaded. If `None` (default) and the
                env variable ``RUBRIX_WORKSPACE`` is not set, it will default to the private user workspace.
            timeout: Wait `timeout` seconds for the connection to timeout. Default: 60.
            extra_headers: Extra HTTP headers sent to the server. You can use this to customize
                the headers of Rubrix client requests, like additional security restrictions. Default: `None`.

        Examples:
            >>> import rubrix as rb
            >>> rb.init(api_url="http://localhost:9090", api_key="4AkeAPIk3Y")
            >>> # Customizing request headers
            >>> headers = {"X-Client-id":"id","X-Secret":"secret"}
            >>> rb.init(api_url="http://localhost:9090", api_key="4AkeAPIk3Y", extra_headers=headers)

        """
        api_url = api_url or os.getenv("RUBRIX_API_URL", "http://localhost:6900")
        # Checking that the api_url does not end in '/'
        api_url = re.sub(r"\/$", "", api_url)
        api_key = api_key or os.getenv("RUBRIX_API_KEY", DEFAULT_API_KEY)
        workspace = workspace or os.getenv("RUBRIX_WORKSPACE")
        headers = extra_headers or {}

        self._client: AuthenticatedClient = AuthenticatedClient(
            base_url=api_url,
            token=api_key,
            timeout=timeout,
            headers=headers.copy(),
        )

        self._user: User = users_api.whoami(client=self._client)
        if workspace is not None:
            self.set_workspace(workspace)

        self._agent = _RubrixLogAgent(self)

    def __del__(self):
        if hasattr(self, "_client"):
            del self._client
        if hasattr(self, "_agent"):
            del self._agent

    @property
    def client(self):
        """The underlying authenticated client"""
        return self._client

    @property
    def datasets(self) -> Datasets:
        return Datasets(client=self._client)

    @property
    def searches(self):
        return Searches(client=self._client)

    @property
    def metrics(self):
        return MetricsAPI(client=self.client)

    def set_workspace(self, workspace: str):
        """Sets the active workspace.

        Args:
            workspace: The new workspace
        """
        if workspace is None:
            raise Exception("Must provide a workspace")

        if workspace != self.get_workspace():
            if workspace == self._user.username:
                self._client.headers.pop(RUBRIX_WORKSPACE_HEADER_NAME, workspace)
            elif (
                self._user.workspaces is not None
                and workspace not in self._user.workspaces
            ):
                raise Exception(f"Wrong provided workspace {workspace}")
            self._client.headers[RUBRIX_WORKSPACE_HEADER_NAME] = workspace

    def get_workspace(self) -> str:
        """Returns the name of the active workspace.

        Returns:
            The name of the active workspace as a string.
        """
        return self._client.headers.get(
            RUBRIX_WORKSPACE_HEADER_NAME, self._user.username
        )

    def copy(self, dataset: str, name_of_copy: str, workspace: str = None):
        """Creates a copy of a dataset including its tags and metadata

        Args:
            dataset: Name of the source dataset
            name_of_copy: Name of the copied dataset
            workspace: If provided, dataset will be copied to that workspace

        Examples:
            >>> import rubrix as rb
            >>> rb.copy("my_dataset", name_of_copy="new_dataset")
            >>> rb.load("new_dataset")
        """
        datasets_api.copy_dataset(
            client=self._client,
            name=dataset,
            json_body=CopyDatasetRequest(name=name_of_copy, target_workspace=workspace),
        )

    def delete(self, name: str) -> None:
        """Deletes a dataset.

        Args:
            name: The dataset name.

        Examples:
            >>> import rubrix as rb
            >>> rb.delete(name="example-dataset")
        """
        datasets_api.delete_dataset(client=self._client, name=name)

    def log(
        self,
        records: Union[Record, Iterable[Record], Dataset],
        name: str,
        tags: Optional[Dict[str, str]] = None,
        metadata: Optional[Dict[str, Any]] = None,
        chunk_size: int = 500,
        verbose: bool = True,
        background: bool = False,
    ) -> Union[BulkResponse, Future]:
        """Logs Records to Rubrix.

        The logging happens asynchronously in a background thread.

        Args:
            records: The record, an iterable of records, or a dataset to log.
            name: The dataset name.
            tags: A dictionary of tags related to the dataset.
            metadata: A dictionary of extra info for the dataset.
            chunk_size: The chunk size for a data bulk.
            verbose: If True, shows a progress bar and prints out a quick summary at the end.
            background: If True, we will NOT wait for the logging process to finish and return an ``asyncio.Future``
                object. You probably want to set ``verbose`` to False in that case.

        Returns:
            Summary of the response from the REST API.
            If the ``background`` argument is set to True, an ``asyncio.Future`` will be returned instead.

        Examples:
            >>> import rubrix as rb
            >>> record = rb.TextClassificationRecord(
            ...     text="my first rubrix example",
            ...     prediction=[('spam', 0.8), ('ham', 0.2)]
            ... )
            >>> rb.log(record, name="example-dataset")
            1 records logged to http://localhost:6900/datasets/rubrix/example-dataset
            BulkResponse(dataset='example-dataset', processed=1, failed=0)
            >>>
            >>> # Logging records in the background
            >>> rb.log(record, name="example-dataset", background=True, verbose=False)
            <Future at 0x7f675a1fffa0 state=pending>
        """
        future = self._agent.log(
            records=records,
            name=name,
            tags=tags,
            metadata=metadata,
            chunk_size=chunk_size,
            verbose=verbose,
        )
        if background:
            return future

        try:
            return future.result()
        finally:
            future.cancel()

    async def log_async(
        self,
        records: Union[Record, Iterable[Record], Dataset],
        name: str,
        tags: Optional[Dict[str, str]] = None,
        metadata: Optional[Dict[str, Any]] = None,
        chunk_size: int = 500,
        verbose: bool = True,
    ) -> BulkResponse:
        """Logs Records to Rubrix with asyncio.

        Args:
            records: The record, an iterable of records, or a dataset to log.
            name: The dataset name.
            tags: A dictionary of tags related to the dataset.
            metadata: A dictionary of extra info for the dataset.
            chunk_size: The chunk size for a data bulk.
            verbose: If True, shows a progress bar and prints out a quick summary at the end.

        Returns:
            Summary of the response from the REST API

        Examples:
            >>> # Log asynchronously from your notebook
            >>> import asyncio
            >>> import rubrix as rb
            >>> from rubrix.utils import setup_loop_in_thread
            >>> loop, _ = setup_loop_in_thread()
            >>> future_response = asyncio.run_coroutine_threadsafe(
            ...     rb.log_async(my_records, dataset_name), loop
            ... )
        """
        tags = tags or {}
        metadata = metadata or {}

        if not name:
            raise InputValueError("Empty dataset name has been passed as argument.")

        if not re.match(DATASET_NAME_REGEX_PATTERN, name):
            raise InputValueError(
                f"Provided dataset name {name} does not match the pattern {DATASET_NAME_REGEX_PATTERN}. "
                "Please, use a valid name for your dataset"
            )

        if chunk_size > self._MAX_CHUNK_SIZE:
            _LOGGER.warning(
                """The introduced chunk size is noticeably large, timeout errors may occur.
                Consider a chunk size smaller than %s""",
                self._MAX_CHUNK_SIZE,
            )

        if isinstance(records, Record.__args__):
            records = [records]
        records = list(records)

        try:
            record_type = type(records[0])
        except IndexError:
            raise InputValueError("Empty record list has been passed as argument.")

        if record_type is TextClassificationRecord:
            bulk_class = TextClassificationBulkData
            creation_class = CreationTextClassificationRecord
        elif record_type is TokenClassificationRecord:
            bulk_class = TokenClassificationBulkData
            creation_class = CreationTokenClassificationRecord
        elif record_type is Text2TextRecord:
            bulk_class = Text2TextBulkData
            creation_class = CreationText2TextRecord
        else:
            raise InputValueError(
                f"Unknown record type {record_type}. Available values are {Record.__args__}"
            )

        processed, failed = 0, 0
        progress_bar = tqdm(total=len(records), disable=not verbose)
        for i in range(0, len(records), chunk_size):
            chunk = records[i : i + chunk_size]

            response = await async_bulk(
                client=self._client,
                name=name,
                json_body=bulk_class(
                    tags=tags,
                    metadata=metadata,
                    records=[creation_class.from_client(r) for r in chunk],
                ),
            )

            processed += response.parsed.processed
            failed += response.parsed.failed

            progress_bar.update(len(chunk))
        progress_bar.close()

        # TODO: improve logging policy in library
        if verbose:
            _LOGGER.info(
                f"Processed {processed} records in dataset {name}. Failed: {failed}"
            )
            workspace = self.get_workspace()
            if (
                not workspace
            ):  # Just for backward comp. with datasets with no workspaces
                workspace = "-"
            print(
                f"{processed} records logged to {self._client.base_url}/datasets/{workspace}/{name}"
            )

        # Creating a composite BulkResponse with the total processed and failed
        return BulkResponse(dataset=name, processed=processed, failed=failed)

    def delete_records(
        self,
        name: str,
        query: Optional[str] = None,
        ids: Optional[List[Union[str, int]]] = None,
        discard_only: bool = False,
        discard_when_forbidden: bool = True,
    ) -> Tuple[int, int]:
        """Delete records from a Rubrix dataset.

        Args:
            name: The dataset name.
            query: An ElasticSearch query with the `query string syntax
                <https://rubrix.readthedocs.io/en/stable/guides/queries.html>`_
            ids: If provided, deletes dataset records with given ids.
            discard_only: If `True`, matched records won't be deleted. Instead, they will be marked as `Discarded`
            discard_when_forbidden: Only super-user or dataset creator can delete records from a dataset.
                So, running "hard" deletion for other users will raise an `ForbiddenApiError` error.
                If this parameter is `True`, the client API will automatically try to mark as ``Discarded``
                records instead. Default, `True`

        Returns:
            The total of matched records and real number of processed errors. These numbers could not
            be the same if some data conflicts are found during operations (some matched records change during
            deletion).

        Examples:
            >>> ## Delete by id
            >>> import rubrix as rb
            >>> rb.delete_records(name="example-dataset", ids=[1,3,5])
            >>> ## Discard records by query
            >>> import rubrix as rb
            >>> rb.delete_records(name="example-dataset", query="metadata.code=33", discard_only=True)
        """
        return self.datasets.delete_records(
            name=name,
            query=query,
            ids=ids,
            mark_as_discarded=discard_only,
            discard_when_forbidden=discard_when_forbidden,
        )

    def load(
        self,
        name: str,
        query: Optional[str] = None,
        ids: Optional[List[Union[str, int]]] = None,
        limit: Optional[int] = None,
        id_from: Optional[str] = None,
        as_pandas=None,
    ) -> Dataset:
        """Loads a Rubrix dataset.

        Parameters:
        -----------
            name:
                The dataset name.
            query:
                An ElasticSearch query with the
                `query string syntax <https://rubrix.readthedocs.io/en/stable/guides/queries.html>`_
            ids:
                If provided, load dataset records with given ids.
            limit:
                The number of records to retrieve.

            id_from:
                If provided, starts gathering the records starting from that Record. As the Records returned with the
                load method are sorted by ID, ´id_from´ can be used to load using batches.

            as_pandas:
                DEPRECATED! To get a pandas DataFrame do ``rb.load('my_dataset').to_pandas()``.

        Returns:
        --------
            A Rubrix dataset.

        Examples:
            **Basic Loading**: load the samples sorted by their ID
            >>> import rubrix as rb
            >>> dataset = rb.load(name="example-dataset")

            **Iterate over a large dataset:**
                When dealing with a large dataset you might want to load it in batches to optimize memory consumption
                and avoid network timeouts. To that end, a simple batch-iteration over the whole database can be done
                employing the `from_id` parameter. This parameter will act as a delimiter, retrieving the N items after
                the given id, where N is determined by the `limit` parameter. **NOTE** If
                no `limit` is given the whole dataset after that ID will be retrieved.

            >>> import rubrix as rb
            >>> dataset_batch_1 = rb.load(name="example-dataset", limit=1000)
            >>> dataset_batch_2 = rb.load(name="example-dataset", limit=1000, id_from=dataset_batch_1[-1].id)

        """
        if as_pandas is False:
            warnings.warn(
                "The argument `as_pandas` is deprecated and will be removed in a future version. "
                "Please adapt your code accordingly. ",
                FutureWarning,
            )
        elif as_pandas is True:
            raise ValueError(
                "The argument `as_pandas` is deprecated and will be removed in a future version. "
                "Please adapt your code accordingly. ",
                "If you want a pandas DataFrame do `rb.load('my_dataset').to_pandas()`.",
            )

        response = datasets_api.get_dataset(client=self._client, name=name)
        task = response.parsed.task

        task_config = {
            TaskType.text_classification: (
                text_classification_api.data,
                TextClassificationQuery,
                DatasetForTextClassification,
            ),
            TaskType.token_classification: (
                token_classification_api.data,
                TokenClassificationQuery,
                DatasetForTokenClassification,
            ),
            TaskType.text2text: (
                text2text_api.data,
                Text2TextQuery,
                DatasetForText2Text,
            ),
        }

        try:
            get_dataset_data, request_class, dataset_class = task_config[task]
        except KeyError:
            raise ValueError(
                f"Load method not supported for the '{task}' task. Supported tasks: "
                f"{[TaskType.text_classification, TaskType.token_classification, TaskType.text2text]}"
            )
        response = get_dataset_data(
            client=self._client,
            name=name,
            request=request_class(ids=ids, query_text=query),
            limit=limit,
            id_from=id_from,
        )

        records = [sdk_record.to_client() for sdk_record in response.parsed]
        try:
            records_sorted_by_id = sorted(records, key=lambda x: x.id)
        # record ids can be a mix of int/str -> sort all as str type
        except TypeError:
            records_sorted_by_id = sorted(records, key=lambda x: str(x.id))

        return dataset_class(records_sorted_by_id)

    def dataset_metrics(self, name: str) -> List[MetricInfo]:
        response = datasets_api.get_dataset(self._client, name)
        response = metrics_api.get_dataset_metrics(
            self._client, name=name, task=response.parsed.task
        )

        return response.parsed

    def get_metric(self, name: str, metric: str) -> Optional[MetricInfo]:
        metrics = self.dataset_metrics(name)
        for metric_ in metrics:
            if metric_.id == metric:
                return metric_

    def compute_metric(
        self,
        name: str,
        metric: str,
        query: Optional[str] = None,
        interval: Optional[float] = None,
        size: Optional[int] = None,
    ) -> MetricResults:
        response = datasets_api.get_dataset(self._client, name)

        metric_ = self.get_metric(name, metric=metric)
        assert metric_ is not None, f"Metric {metric} not found !!!"

        response = metrics_api.compute_metric(
            self._client,
            name=name,
            task=response.parsed.task,
            metric=metric,
            query=query,
            interval=interval,
            size=size,
        )

        return MetricResults(**metric_.dict(), results=response.parsed)

    def fetch_dataset_labeling_rules(self, dataset: str) -> List[LabelingRule]:
        response = text_classification_api.fetch_dataset_labeling_rules(
            self._client, name=dataset
        )

        return [LabelingRule.parse_obj(data) for data in response.parsed]

    def rule_metrics_for_dataset(
        self, dataset: str, rule: LabelingRule
    ) -> LabelingRuleMetricsSummary:
        response = text_classification_api.dataset_rule_metrics(
            self._client, name=dataset, query=rule.query, label=rule.label
        )

        return LabelingRuleMetricsSummary.parse_obj(response.parsed)

__ACTIVE_API__: Optional[Api] = None

def active_api() -> Api:
    """Returns the active API.

    If Active API is None, initialize a default one.
    """
    global __ACTIVE_API__
    if __ACTIVE_API__ is None:
        __ACTIVE_API__ = Api()
    return __ACTIVE_API__

def api_wrapper(api_method: Callable):
    """Decorator to wrap the API methods in module functions.

    Propagates the docstrings and adapts the signature of the methods.
    """

    def decorator(func):
        if asyncio.iscoroutinefunction(api_method):

            @wraps(api_method)
            async def wrapped_func(*args, **kwargs):
                return await func(*args, **kwargs)

        else:

            @wraps(api_method)
            def wrapped_func(*args, **kwargs):
                return func(*args, **kwargs)

        sign = signature(api_method)
        wrapped_func.__signature__ = sign.replace(
            parameters=[val for key, val in sign.parameters.items() if key != "self"]
        )
        return wrapped_func

    return decorator

@api_wrapper(Api.__init__)
def init(*args, **kwargs):
    global __ACTIVE_API__
    __ACTIVE_API__ = Api(*args, **kwargs)

@api_wrapper(Api.set_workspace)
def set_workspace(*args, **kwargs):
    return active_api().set_workspace(*args, **kwargs)

@api_wrapper(Api.get_workspace)
def get_workspace(*args, **kwargs):
    return active_api().get_workspace(*args, **kwargs)

@api_wrapper(Api.copy)
def copy(*args, **kwargs):
    return active_api().copy(*args, **kwargs)

@api_wrapper(Api.delete)
def delete(*args, **kwargs):
    return active_api().delete(*args, **kwargs)

@api_wrapper(Api.log)
def log(*args, **kwargs):
    return active_api().log(*args, **kwargs)

@api_wrapper(Api.log_async)
def log_async(*args, **kwargs):
    return active_api().log_async(*args, **kwargs)

@api_wrapper(Api.load)
def load(*args, **kwargs):
    return active_api().load(*args, **kwargs)

@api_wrapper(Api.delete_records)
def delete_records(*args, **kwargs):
    return active_api().delete_records(*args, **kwargs)

class InputValueError(RubrixClientError):
    pass


#!/home/shailja/.virtualenv/my_env/bin/python3

import requests
import bs4
import sys

content =  sys.argv[1]

def display_actual_text(text,para_no):
    text = text[para_no]
    [s.extract() for s in text(['style', 'script', '[document]', 'head', 'title'])]
    visible_text = text.getText()
    print(visible_text)

wiki_url = 'https://en.wikipedia.org/wiki/{}'

request = requests.get(wiki_url.format(content))
soup = bs4.BeautifulSoup(request.text,'lxml')
actual_text=soup.select('p')

# this is to handle if wikipedia don't have any article with this name
try:
    display_actual_text(actual_text,1)
except:
    print(f"Wikipedia does not have an article with this exact name.")
    exit(1)

    # wiki_url = 'https://en.wikipedia.org/wiki/Special:Search?go=Go&search={}&ns0=1'

    # request = requests.get(wiki_url.format(content))
    # soup = bs4.BeautifulSoup(request.text,'lxml')
    # #relevant_text=soup.select("div", {"id": "bodyContent"})                          #soup.select('#mw-body')
    # #relevant_text=relevant_text.select("div.searchresults")
    # content = soup.select(".mw-search-result")[0].select('td')[1]
    # title=content.a['title']
    # relevant_text=content.select(".searchresult")[0]
    # [s.extract() for s in relevant_text(['style', 'script', '[document]', 'head', 'title'])]
    # visible_text = relevant_text.getText()
    # print(title)
    # print(visible_text)

# for more data

show_next = False
para_no = 2

# this is to handle if search is found and reaches to the end of content in wikipidea
try:
    while not show_next:
        show_next = input("!")
        print()
        display_actual_text(actual_text,para_no)
        para_no+=1

except:
    print("End")


from mlagents_envs.environment import UnityEnvironment
from mlagents_envs.environment import ActionTuple
from mlagents_envs.side_channel.engine_configuration_channel import EngineConfigurationChannel

import numpy as np
import mlagents.trainers

from collections import namedtuple

obs = namedtuple(
        'obs',
        ['vector', 'front', 'right', 'back', 'left', 'raycast'])

class Drone(object):

    def __init__(self, time_scale=1.0, filename='mac.app', port=11000):

        self.engine_configuration_channel = EngineConfigurationChannel()
        print(f"VERSION : {mlagents.trainers.__version__}")
        self.env = UnityEnvironment(
                file_name=filename,
                worker_id=port,
                side_channels=[self.engine_configuration_channel])
        self.env.reset()

        self.behavior_name = list(self.env.behavior_specs.keys())[0]
        self.spec = self.env.behavior_specs[self.behavior_name]
        self.engine_configuration_channel.set_configuration_parameters(time_scale=time_scale)
        self.dec, self.term = self.env.get_steps(self.behavior_name)

    def reset(self):

        self.env.reset()
        self.dec, self.term = self.env.get_steps(self.behavior_name)
        self.tracked_agent = -1

        self.state = [self.dec.obs[i][0] for i in range(6)]
        self.state = obs(
                vector=self.state[0], front=self.state[1],
                right=self.state[2], back=self.state[3],
                left=self.state[4], raycast=self.state[5])

        return self.state

    def step(self, action):

        if self.tracked_agent == -1 and len(self.dec) >= 1:
            self.tracked_agent = self.dec.agent_id[0]

        action = np.clip(action, -1, 1)
        action_tuple = ActionTuple()
        action_tuple.add_continuous(np.array([action]))

        self.env.set_actions(self.behavior_name, action_tuple)
        self.env.step()

        self.dec, self.term = self.env.get_steps(self.behavior_name)

        reward = 0
        done = False
        if self.tracked_agent in self.dec:
            reward += self.dec[self.tracked_agent].reward
        if self.tracked_agent in self.term:
            reward += self.term[self.tracked_agent].reward
            done = True

        if done:
            return self.state, reward, done

        self.state = [self.dec.obs[i][0] for i in range(6)]
        self.state = obs(
                vector=self.state[0], front=self.state[1],
                right=self.state[2], back=self.state[3],
                left=self.state[4], raycast=self.state[5])

        return self.state, reward, done

if __name__ == '__main__':

    import matplotlib.pyplot as plt

    env = Drone(
            time_scale=0.1,
            filename='/Users/chageumgang/Desktop/baselines/mac.app')

    episode = 0
    while True:

        state = env.reset()
        done = False
        score = 0
        episode += 1

        while not done:

            action = np.random.rand(3)
            next_state, reward, done = env.step(action)
            score += reward

            print(next_state.vector.shape)
            print(next_state.raycast.shape)
            print(next_state.front.shape)

            '''
            fig = plt.figure()
            ax1 = fig.add_subplot(2, 2, 1)
            ax1.imshow(state.front)
            ax1 = fig.add_subplot(2, 2, 2)
            ax1.imshow(state.back)
            ax1 = fig.add_subplot(2, 2, 3)
            ax1.imshow(state.right)
            ax1 = fig.add_subplot(2, 2, 4)
            ax1.imshow(state.left)
            plt.show(block=False)
            plt.pause(0.1)
            '''

            state = next_state

        print(episode, score)


import pandas as pd
from bs4 import BeautifulSoup as bs

#Criando objeto BS
def get_file(file_name):
    content = []
    with open(file_name, 'r') as file:
        content = file.readlines()
        content = ''.join(content)
        soup = bs(content,'xml')
    return soup

#Buscando parents
def get_parents(soup):
    parents = []
    for p in soup.parents:
        if p.has_attr("NAME") and p.find('TRANSFORMATION'):
            parents.append(p["NAME"])
    return parents

#Buscando valores
def get_values(soup, tag1):
    mapping = soup.find_all(tag1)
    names = []
    descriptions = []
    parents = []
    for map in mapping:
        names.append(map.get('NAME'))
        descriptions.append(map.get('DESCRIPTION'))
        parents.append(get_parents(map)) #soup>map
    list_values = [parents, names, descriptions]
    return list_values

#Criando DF
def create_df(list_values):
    dict = {
        'PARENTS': list_values[0],
        'NAME': list_values[1],
        'DESCRIPTION': list_values[2]
    }
    df = pd.DataFrame(dict)
    return df

#Criando excel
def create_file(filename, df):
    return df.to_excel(excel_writer=filename, sheet_name=filename, header=True)

#Separando colunas
def transform_column_parents(file):
    df_origin = pd.read_excel(file)
    list_column = df_origin['PARENTS'].values.tolist()
    list_column = [eval(item) for item in list_column]
    try:
        df_column = pd.DataFrame(list_column, columns=['MAPPING', 'FOLDER', 'REPOSITORY'])
        df_clean = df_origin.drop(['PARENTS'], axis=1)
        df_clean = df_origin.drop(['Unnamed: 0'], axis=1)
        df_complete = pd.concat([df_column, df_clean], axis=1)
    except:
        df_column = pd.DataFrame(list_column, columns=['FOLDER', 'REPOSITORY'])
        df_clean = df_origin.drop(['PARENTS'], axis=1)
        df_clean = df_origin.drop(['Unnamed: 0'], axis=1)
        df_complete = pd.concat([df_column, df_clean], axis=1)
    else:
        print('working')
        df_clean = df_origin.drop(['PARENTS'], axis=1)
        df_clean = df_origin.drop(['Unnamed: 0'], axis=1)
        df_complete = pd.concat([df_column, df_clean], axis=1)
    return df_complete

if __name__ == '__main__':
    #Strings
    name = '../mapping.XML'
    tag_mapping = 'MAPPING'
    tag_transformation = 'TRANSFORMATION'
    tag_target = 'TARGET'
    tag_source = 'SOURCE'
    filename_mapping = 'mapping.xlsx'
    filename_transformation = 'transformation.xlsx'
    filename_target = 'target.xlsx'
    filename_source = 'source.xlsx'

    #Criando objeto soup
    soup = get_file(name)

    #buscando valores de mapping, criando df e criando xlsx
    list_mapping = get_values(soup, tag_mapping)
    df_mapping = create_df(list_mapping)
    create_file(filename_mapping, df_mapping)

    #buscando valores de transformation, criando df e criando xlsx
    list_transformation = get_values(soup, tag_transformation)
    df_transformation = create_df(list_transformation)
    create_file(filename_transformation, df_transformation)

    #buscando valores de target, criando df e criando xlsx
    list_target = get_values(soup, tag_target)
    df_target = create_df(list_target)
    create_file(filename_target, df_target)

    #buscando valores de source, criando df e criando xlsx
    list_source = get_values(soup, tag_source)
    df_source = create_df(list_source)
    create_file(filename_source, df_source)

    #Bloco para dividir em tabelas os parents
    file1 = '../source.xlsx'
    file2 = '../mapping.xlsx'
    file3 = '../transformation.xlsx'
    file4 = '../target.xlsx'

    #Sources
    df1 = transform_column_parents(file1)
    create_file(df1, 'source.xlsx')

    #Mappings
    df2 = transform_column_parents(file2)
    create_file(df2, 'mapping.xlsx')

    #Transformations
    df3 = transform_column_parents(file3)
    create_file(df3, 'transformation.xlsx')

    #Targets
    df4 = transform_column_parents(file4)
    create_file(df4, 'target.xlsx')


#!/bin/python3

import math
import os
import random
import re
import sys
from collections import deque

# Complete the bfs function below.
def bfs(n, m, edges, s):
    #Create adjacency list empty on array
    neighbors = [[] for i in range(n) for j in range(1)]
    #Include neighbors of each vertex, with index minus 1
    for e in edges:
        neighbors[e[0]-1].append(e[1]-1)
        neighbors[e[1]-1].append(e[0]-1)

    #Seen and Distance begin in False and -1 each
    seen = [False for i in range(n)]
    distance = [-1 for i in range(n)]

    #Start vertex as True and distance 0
    seen[s - 1] = True
    distance[s - 1] = 0
    print(seen)
    print(distance)

    q = deque()
    q.append(s - 1)

    while len(q):
        v = q.popleft()
        for u in neighbors[v]:
            if seen[u] == False:
                distance[u] = distance[v] + 6
                seen[u] = True
                q.append(u)
        seen[v] = True

    return [distance[i] for i in range(n) if i != s-1]

if __name__ == '__main__':
    fptr = open(os.environ['OUTPUT_PATH'], 'w')

    q = int(input())

    for q_itr in range(q):
        nm = input().split()

        n = int(nm[0])

        m = int(nm[1])

        edges = []

        for _ in range(m):
            edges.append(list(map(int, input().rstrip().split())))

        s = int(input())

        result = bfs(n, m, edges, s)

        fptr.write(' '.join(map(str, result)))
        fptr.write('\n')

    fptr.close()


#imports
import numpy as np
import matplotlib.pyplot as plt
import scipy.constants as const
from scipy.special import iv as I0
from scipy.special import kv as K0

#Define Global Variables
L_geo = 55.6e-9
Z0 = 50.0
F0_base = 0.95e9        #At lowest Temp
squares= 27223
c_couple = 1.5e-14
TC = 1.5
Delta_0 = (3.5*const.Boltzmann*TC)/2
sigma_n = 6.0e7          # Normal stae conductvity if superconducting film
Thick = 20e-9            # Thickness of superconducting fil
w = 2 * np.pi * F0_base
me = const.m_e
miu_0 = 4*np.pi*10**-7
pi = np.pi

#Main code
def main():
    #Define temperature range with step 0.01K
    step = 0.1
    temp = np.arange(0.2, 0.3   , step)

    #Find sigma1 and sigma 2 and Lint
    sigma1, sigma2 = find_sigma1_sigma2(sigma_n ,Thick, TC, Delta_0, w, temp)
    Lint = find_Lint_square(Thick, w, sigma2) * squares

    #Find lk
    Lk = find_lk(Thick, w, sigma2)

    #Find Res
    sigma12Ratio = sigma1/sigma2
    Res = Lk*w*sigma12Ratio *squares

    #IDC for Lowest Temp (0.2K)
    Ltot_lowest = Lint[0] + L_geo
    IDC = find_IDC(w, Ltot_lowest, c_couple)

    #Find S21
    Sweep_points = 20000
    BW = 5e6
    I_raw = np.zeros((Sweep_points, len(temp)), dtype="float")
    Q_raw = np.copy(I_raw)
    Phase = np.copy(Q_raw)
    S21_Volt = np.copy(I_raw)
    for i in range(0, len(Lint)):
        Sweep, S21_Volt[:,i], Phase[:,i], I_raw[:,i], Q_raw[:,i],_,_,_,_,_ = Capacitive_Res_Sim(F0_base, c_couple, Z0, L_geo, Lint[i], Res[i], BW, Sweep_points, IDC)
        plt.plot(Sweep/1e9, S21_Volt[:,i], label=str("{:.2f}".format(temp[i])))

    #Graph labels and title
    plt.legend(loc='center left', bbox_to_anchor=(1, 0.5), fancybox=True, title="Temperature / K")
    plt.xlabel('Frequency / GHz', fontsize=13)

    plt.ylabel('Output Amplitude', fontsize=13)
    #plt.title("S21 Amplitude For Varying Temperatures")
    plt.xlim(0.9490, 0.9505)
    plt.locator_params(nbins=6)
    plt.savefig("S21 Plot with Resistance")
    plt.rcParams['figure.dpi'] = 300
    plt.figure()

    #Q vs I plots
    for i in range(0, len(Lint)):
        plt.plot(I_raw[:,i], Q_raw[:,i], linewidth=1,label=str("{:.2f}".format(temp[i])))

    #Minimum S21 at lowest temp
    S21_Base = min(S21_Volt[:,0])
    I_Base = np.zeros(len(temp), dtype="float")
    Q_Base = np.copy(I_Base)

    #Obtain F0_base and I and Q values for Lowest Temp
    for i in range(0, len(S21_Volt[:,0])):
        if S21_Base == S21_Volt[i,0]:
            F0_Base = Sweep[i]

    #Plot I and Q values at F0_Base
    for i in range(0, len(temp)):
        for j in range(0, len(Sweep)):
            if F0_Base == Sweep[j]:
                I_Base[i] = I_raw[j,i]
                Q_Base[i] = Q_raw[j,i]
                plt.plot(I_Base[i], Q_Base[i], markersize=4, marker="x", color='black')

    #labels
    plt.legend(loc='center left', bbox_to_anchor=(1, 0.5), fancybox=True, title="Temperature / K")
    plt.xlabel('I / V', fontsize=13)
    plt.ylabel('Q / V', fontsize=13)
    #plt.title("Q vs I Plot for Varying Temperature")
    plt.savefig("Q vs I plot for varying temp")
    plt.figure()

    #Finding F0 for the different Temperatures
    F0 = np.zeros(len(temp))
    for i in range(0, len(temp)):
        S21_min = min(S21_Volt[:,i])
        for j in range(0, len(Sweep)):
            if S21_min == S21_Volt[j,i]:
                F0[i] = Sweep[j]

    #Plotting F0 vs Temp
    plt.plot(temp, F0/1e9, color='k', linewidth="1", label="Minimum Of S21")
    plt.xlabel('Temperature / K', fontsize=13)
    plt.ylabel('F0 / GHz', fontsize=13)
    plt.rcParams['figure.dpi'] = 300
    #plt.title("F0 vs Temperature")

    #Finding dI/dF and dQ/dF for lowest temperature
    #Using numerical derivatives
    step = abs((Sweep[0]-Sweep[-1])/Sweep_points)
    for i in range(0, len(Sweep)):
        if Sweep[i] == F0_Base:
            didf = (I_raw[i+1,0] - I_raw[i-1,0])/(2*step)
            dqdf = (Q_raw[i+1,0] - Q_raw[i-1,0])/(2*step)

    #Use Magic Formula
    di = np.zeros(len(temp))
    dq = np.copy(di)
    di = abs(I_Base - I_Base[0])
    dq = abs(Q_Base - Q_Base[0])
    dF0 = Magic_Formula(di, dq, didf, dqdf)

    #Find F0 for different temp
    F0_Magic = F0_Base - abs(dF0)
    plt.plot(temp, F0_Magic/1e9, label="dF0 Formula")
    plt.legend(loc='center left', bbox_to_anchor=(1, 0.5), fancybox=True)
    plt.ticklabel_format(useOffset=False)
    plt.rcParams['figure.dpi'] = 1000
    plt.xlim(0.20, 0.22)
    plt.ylim(0.949980, 0.95)
    plt.savefig("Magic Formula plot")

#KID Simulating Function
def Capacitive_Res_Sim(F0, C_couple, Z0, L_geo, L_int, Res, Sweep_BW, Sweep_points, Capacitance):
    """ Help file here"""
    j=complex(0,1)
    Cc=C_couple
    F_min=F0-(Sweep_BW/2.0)
    F_max=F0+(Sweep_BW/2.0)
    Sweep=np.linspace(F_min, F_max, Sweep_points)
    W=Sweep*2.0*pi
    W0=2.0*pi*F0
    L=L_geo+L_int
    C=Capacitance
    Zres= 1.0/((1./((j*W*L)+Res))+(j*W*C))   # Impedance of resonator section
    Zc=1.0/(j*W*Cc)  #impedance of coupler
    ZT=Zres+Zc
    YT=1.0/ZT
    S21 = 2.0/(2.0+(YT*Z0))
    I_raw=S21.real
    Q_raw=S21.imag
    shift=((1.0-min(I_raw))/2.0)+min(I_raw)
    I_cent=I_raw-shift
    Q_cent=Q_raw
    Phase=Atan(abs(Q_cent/I_cent))
    QU=(W0*L)/Res
    QL=(C*2)/(W0*(Cc**2)*Z0)
    S21_Volt=abs(S21)
    I_offset=shift
    return  (Sweep, S21_Volt, Phase, I_raw, Q_raw, I_cent, Q_cent, QU, QL, I_offset)

#Function to find sigma1 and sigma2
def find_sigma1_sigma2(sigma_n ,Thick, TC, Delta_0, w, T):
    #An interpolation formula for delta_T
    delta_T = Delta_0*np.tanh(1.74*np.sqrt((TC/T)-1))

    #Define constants to simplify eqn
    multiplying_constant = delta_T/(const.hbar * w)
    e_const_1 = - Delta_0/(const.Boltzmann*T)
    e_const_2 = (const.hbar*w)/(2*const.Boltzmann*T)

    #Parts of the sigma1 Ratio
    A = 2*multiplying_constant
    B = np.exp(e_const_1)
    C = K0(0, e_const_2)
    D = 2*(np.sinh(e_const_2))

    #Find Sigma 1 and Sigma 2
    sigma1Ratio = A * B * C * D
    sigma2Ratio = np.pi*multiplying_constant*(1 - (2*np.exp(e_const_1)*np.exp(-e_const_2)*I0(0,e_const_2)))
    sigma2 = sigma2Ratio * sigma_n
    sigma1 = sigma1Ratio * sigma_n
    return sigma1, sigma2

def find_lk(Thick, w, sigma2):
    #Depth
    lower_fraction = miu_0*sigma2*w
    Lambda_T_MB = (1/lower_fraction)**0.5
    fraction = Thick/(2*Lambda_T_MB)

    #Terms for lk
    A = (miu_0*Lambda_T_MB)/4
    B = coth(fraction)
    C = fraction*(csch(fraction))**2

    #R vs T
    lk = A*(B+C)
    return lk

def find_Lint_square(Thick, w, sigma2):
    #Depth
    lower_fraction = miu_0*sigma2*w
    Lambda_T_MB = (1/lower_fraction)**0.5

    #Internal Inductance
    fraction = Thick/(2*Lambda_T_MB)
    L_int = (miu_0*Lambda_T_MB/2)*coth(fraction)
    return L_int

#Define coth and csch
def coth(x):
    return np.cosh(x)/np.sinh(x)

def csch(x):
    return 1/np.sinh(x)

def Atan(x):
    return np.arctan(x)

#Find IDC function
def find_IDC(w0, Ltot, Cc):
    IDC = 1/((w0**2)*Ltot) - Cc
    return IDC

def Magic_Formula(di, dq, didf, dqdf):
    return (di*didf + dq*dqdf)/(didf**2 + dqdf**2)

main()


# from urllib import request
from django.shortcuts import render, redirect
from .models import Post, Comment
from .forms import CommentForm, PostUpdateForm
# from django.http import HttpResponseRedirect
from django.contrib.auth.decorators import login_required

# LoginRequiredMixin is simply the class based version for login_required decorator.
# because we cannot use decorators with classes, we are using mixins instead.
# UserPassesTestMixin mixin is for making sure that only the author can edit the posts
from django.contrib.auth.mixins import (LoginRequiredMixin,
    UserPassesTestMixin)
from django.views.generic import (ListView,
    # DetailView,
    CreateView,
    UpdateView,
    DeleteView)

#! FBV for listing blog posts
def home(request):
    context = {
        'posts': Post.objects.all()
    }
    return render(request, 'blog/home.html', context)

#! CBV for listing blog posts
class PostListView(ListView):
    model = Post
    template_name = 'blog/home.html' # if I had left this field empty, django would have looked for 'blog/post_list' template cuz it looks for <app>/<model>_<viewtype.html> by default
    context_object_name = 'posts' # by default, django uses the name "object_list". If this field was left empty, I'd have to use object_list to loop through my posts in home.html
    ordering = ['-date_posted']

#! CBV for individual blog posts
# class PostDetailView(LoginRequiredMixin, DetailView):
#     model = Post
#     form = CommentForm
#     # actually, for this one let's create the defualt html template file django is looking for. And since this
#     # is a detailview, django's gonna look for a template named 'blog/post_detail.html'.
#     # not defining our context_object_name, we'll have to use 'object' for every post in our blog/post_detail.html template

#     #? view count part
#     def get_object(self):
#         views = super().get_object()
#         views.blog_view += 1
#         views.save()
#         print(CommentForm)
#         print(Post.objects.get(id = views.pk) == views)
#         # if request.method == 'POST':
#         #     print(CommentForm(request.POST))
#         # if form.is_valid():
#         #     comment=form.save(commit=False)
#         #     comment.blog = views
#         #     comment.save()

#         return views

#! FBV for individual blog posts
@login_required
def blog_detail(request, pk):
    post = Post.objects.get(id=pk)
    print(post.post_image)
    form = CommentForm()
    form_blog= PostUpdateForm()
    comments = Comment.objects.filter(post=post.id)
    post.blog_view += 1
    post.save()
    if request.method == 'POST':
        form = CommentForm(request.POST)
        print('yay1')
        if form.is_valid():
            print('yay2')
            comment = form.save(commit=False)
            comment.post = post
            post.blog_comment +=1
            comment.user = request.user
            post.blog_view -= 2
            post.save()
            comment.save()
            return redirect("post-detail", pk)
    return render(request, 'blog/post_detail.html', {'post': post, 'form': form, 'comments': comments})

#! CBV for creating blog posts
class PostCreateView(LoginRequiredMixin, CreateView): # make sure you add your mixins to the left. They should be inherited first, in other words
    model = Post
    fields = ('title', 'content', 'post_image')
    success_url = '/'

    # we are getting a "NOT NULL constraint failed: blog_post.author_id" after posting a blog post which
    # means that the post needs an author and django by default cannot know who the author is. Therefore,
    # we'll need to override the form_valid method and set the author before saving it
    def form_valid(self, form):
        form.instance.author = self.request.user
        return super().form_valid(form)

#! CBV for updating blog posts
class PostUpdateView(LoginRequiredMixin, UserPassesTestMixin, UpdateView):
    model = Post
    fields = ('title', 'content', 'post_image')

    def form_valid(self, form):
        form.instance.author = self.request.user
        return super().form_valid(form)

    def test_func(self): # UserPassesTestMixin requires to override the test_func, thus we are defining it here
        post = self.get_object()
        if self.request.user == post.author:
            return True
        return False

#! CBV for deleting blog posts
class PostDeleteView(LoginRequiredMixin, UserPassesTestMixin, DeleteView):
    model = Post
    # fields = ('title', 'content')
    success_url = '/'

    def test_func(self):
        post = self.get_object()
        if self.request.user == post.author:
            return True
        return False

def like_post(request, id):
    if request.method == "POST":
        instance = Post.objects.get(id=id)
        # print(request.user.id, id)
        # print(instance.author)
        # print(instance.likes.id)
        if not instance.likes.filter(id=request.user.id).exists():
            instance.likes.add(request.user)
            instance.save()
            # print(instance)
            return render( request, 'blog/likes_area.html', context={'post':instance})
        else:
            instance.likes.remove(request.user)
            instance.save()
            # print(instance)
            return render( request, 'blog/likes_area.html', context={'post':instance})

# def post_comment(request, id):
#     model = Comment.objects.get(id=id)
#     comment = CommentForm(instance=model)

#     if request.method == "POST":
#         comment = CommentForm(request.POST, instance=model)
#         # instance = CommentForm(request, id=id)
#         if comment.is_valid():
#             comment.save()
#             return render( request, 'blog/post_comment.html', context={'comment':comment})

# class CommentView(CreateView):
#     model = Comment

#     template_name = 'blog/post_comment.html'

#     fields = ('post', 'body','date_added')

def about(request):
    return render(request, 'blog/about.html')


import math
from datetime import datetime

import firebase_admin
from firebase_admin import credentials
from firebase_admin import db
from firebase_admin import firestore

import parseIntervalFiles as pif
import parseActivityFiles as paf

hervdir = "C:\\Users\\Ju\\GDrive\\Projects\\HeRV\\"

## Firestore connection parameters

keyfile = hervdir + "Docs\\herv-3c5ea-firebase-adminsdk-99tjk-98193df3d9.json"
databaseURL = 'https://herv-3c5ea.firebaseio.com'

## CSV file reading parameters

source = hervdir + "Data\\Raw\\"
start_dt = datetime(2017, 10, 29)
end_dt = datetime(2018, 11, 1)

def u_ref(db, uid):
    return db.collection('users').document(str(uid))

def add_sessions(uid, start_dt, end_dt, source, dest):
    u_sess = paf.get_sessions(uid, start_dt, end_dt, source, verbose=False)
    print('adding', len(u_sess), 'sessions for user', uid)
    s_ref = u_ref(dest, uid).collection('sessions')
    b = dest.batch()
    for sess in u_sess:
        name = paf.csvu.string_from_time_filename(sess['start'])
        doc = s_ref.document(name)
        b.set(doc, sess)
    b.commit()

def add_intervals(uid, start_dt, end_dt, source, dest):
    for day in pif.csvu.gendays(start_dt, end_dt):
        add_day_intervals(uid, day, source, dest)
    print ("finished adding intervals for user", uid)

def add_day_intervals(uid, day, source, dest):
    day_rr = pif.get_day_intervals(uid, day, source)
    if len(day_rr) > 0:
        dayname = datetime.strftime(day, "%Y%m%d")
        print(len(day_rr), 'RR intervals in', dayname)
        rr_ref = u_ref(dest, uid).collection('rr')
        rr_ref.document(dayname).set({'rr_count': len(day_rr)})
        mref = rr_ref.document(dayname).collection('minutes')
        for min_batch in batch(group_by_minute(day_rr)):
            print ('adding batch with', len(min_batch), 'minutes')
            gr = dest.batch()
            for (k, v) in min_batch:
                doc = mref.document(k)
                gr.set(doc, v)
            gr.commit()

def batch(d, n=500):
    x = len(d)
    l = list(d.items())
    for ndx in range(0, x, n):
        yield l[ndx:min(ndx + n, x)]

def group_by_minute(dayrr):
    d = {}
    #TODO obviously, this can be done without looping by splitting the list by h,m
    for h in range(24):
        for m in range(60):
            mrr = [x for x in dayrr if x['date'].hour == h and x['date'].minute == m]
            if len(mrr) > 0:
                miname = str(str(h).zfill(2) + str(m).zfill(2))
                mi = {}
                for s in range(60):
                    srr = [x['interval'] for x in mrr if x['date'].second == s]
                    if len(srr) > 0:
                        mi[str(s)] = srr
                d[miname] = mi
    return d

## Initializing a client to communicate with Firestore

cred = credentials.Certificate(keyfile)
default_app = firebase_admin.initialize_app(cred, options={'databaseURL': databaseURL})
client = firestore.client()

print ("Connected to Firestore...")

## for each user id in the database, search for sessions and intervals in csvs

users = client.collection('users')
userlist = [int(doc.id) for doc in users.get()]

for uid in userlist:
    print("\n\nUSER", uid, "\n\n")
    add_sessions(uid, start_dt, end_dt, source, client)
    add_intervals(uid, start_dt, end_dt, source, client)