import os
import sys
sys.path.append('..')
sys.path.append('../..')
import argparse
import utils
from tsp_helper import *
from student_utils import *

OUTPUT_FILENAME = "naive_output.txt"

"""
======================================================================
  Complete the following function.
======================================================================
"""

def solve(list_of_locations, list_of_homes, starting_car_location, adjacency_matrix, params=[]):
    """
    Write your algorithm here.
    Input:
        list_of_locations: A list of locations such that node i of the graph corresponds to name at index i of the list
        list_of_homes: A list of homes
        starting_car_location: The name of the starting location for the car
        adjacency_matrix: The adjacency matrix from the input file
    Output:
        A list of locations representing the car path
        A dictionary mapping drop-off location to a list of homes of TAs that got off at that particular location
        NOTE: both outputs should be in terms of indices not the names of the locations themselves
    """
    # Ensure that the starting location is at the start.
    homes_and_start = list_of_homes.copy()
    if starting_car_location not in list_of_homes:
        homes_and_start.insert(0, starting_car_location)
    else:
        homes_and_start.remove(starting_car_location)
        homes_and_start.insert(0, starting_car_location)

    # Initialize storage structs (see below)
    sp_lookup = {}
    only_tas_adjacency = []

    # Get shortest paths between all houses w. networkx
    G, error = adjacency_matrix_to_graph(adjacency_matrix)
    for i, home1 in enumerate(homes_and_start):
        home1_index = list_of_locations.index(home1)
        home1_row = []
        for j, home2 in enumerate(homes_and_start):
            if home1 == home2:
                home1_row.append(0)
                continue
            home2_index = list_of_locations.index(home2)
            sp = nx.shortest_path(G, source=home1_index, target=home2_index)
            cost = cost_of_path(sp, adjacency_matrix)
            # Construct lookup dict ((i, j), (A, ... , B))
            # i.e. ((int, int), (int, ..., int))
            sp_lookup[(i, j)] = (tuple(sp))

            home1_row.append(cost)
        # Create adjacency mtx of only TA homes and start, for use in ortools tsp solver
        only_tas_adjacency.append(home1_row)

    # Plus into tsp_solver, get back shortest tour of TA houses
    shortest_tour = get_tsp_result(only_tas_adjacency, 0)

    # Replace house paths w actual paths from dict
    car_tour = []
    is_start = True
    first_ind = 0
    for index in range(len(shortest_tour) - 1):
        i, j = shortest_tour[index], shortest_tour[index + 1]
        sp = sp_lookup[(i, j)]
        if is_start:
            first_ind = sp[0]
            is_start = False
        sp_without_last = sp[:len(sp_lookup[(i, j)]) - 1]
        for elem in sp_without_last:
            car_tour.append(elem)
    car_tour.append(first_ind)

    # Each TA gets dropped off at their house.
    dropoff_dict = {}
    for location_ind in car_tour:
        if list_of_locations[location_ind] in list_of_homes:
            dropoff_dict[location_ind] = [location_ind]

    # Done
    print(car_tour)
    print(dropoff_dict)
    return car_tour, dropoff_dict

""" Lookup in adj matrix the cost of some path"""
def cost_of_path(path, adj_matrix):
    total, start, next = 0, 0, 1
    while next < len(path):
        total += adj_matrix[path[start]][path[next]]
        start += 1
        next += 1
    return total

"""
======================================================================
   No need to change any code below this line
======================================================================
"""

"""
Convert solution with path and dropoff_mapping in terms of indices
and write solution output in terms of names to path_to_file + file_number + '.out'
"""
def convertToFile(path, dropoff_mapping, path_to_file, list_locs):
    string = ''
    for node in path:
        string += list_locs[node] + ' '
    string = string.strip()
    string += '\n'

    dropoffNumber = len(dropoff_mapping.keys())
    string += str(dropoffNumber) + '\n'
    for dropoff in dropoff_mapping.keys():
        strDrop = list_locs[dropoff] + ' '
        for node in dropoff_mapping[dropoff]:
            strDrop += list_locs[node] + ' '
        strDrop = strDrop.strip()
        strDrop += '\n'
        string += strDrop
    utils.write_to_file(path_to_file, string)

def solve_from_file(input_file, output_directory, params=[]):
    print('Processing', input_file)

    input_data = utils.read_file(input_file)
    num_of_locations, num_houses, list_locations, list_houses, starting_car_location, adjacency_matrix = data_parser(input_data)
    car_path, drop_offs = solve(list_locations, list_houses, starting_car_location, adjacency_matrix, params=params)

    basename, filename = os.path.split(input_file)
    if not os.path.exists(output_directory):
        os.makedirs(output_directory)
    output_file = utils.input_to_output(input_file, output_directory)

    convertToFile(car_path, drop_offs, output_file, list_locations)

def solve_all(input_directory, output_directory, params=[]):
    input_files = utils.get_files_with_extension(input_directory, 'in')

    for input_file in input_files:
        solve_from_file(input_file, output_directory, params=params)

if __name__=="__main__":
    parser = argparse.ArgumentParser(description='Parsing arguments')
    parser.add_argument('--all', action='store_true', help='If specified, the solver is run on all files in the input directory. Else, it is run on just the given input file')
    parser.add_argument('input', type=str, help='The path to the input file or directory')
    parser.add_argument('output_directory', type=str, nargs='?', default='.', help='The path to the directory where the output should be written')
    parser.add_argument('params', nargs=argparse.REMAINDER, help='Extra arguments passed in')
    args = parser.parse_args()
    output_directory = args.output_directory
    if args.all:
        input_directory = args.input
        solve_all(input_directory, output_directory, params=args.params)
    else:
        input_file = args.input
        solve_from_file(input_file, output_directory, params=args.params)


import torch
import torch.nn as nn
from transformers import BertModel

class SentimentClassifier(nn.Module):

    def __init__(self, freeze_bert = True):
        super(SentimentClassifier, self).__init__()
        #Instantiating BERT model object
        self.bert_layer = BertModel.from_pretrained('bert-base-uncased')

        #Freeze bert layers
        if freeze_bert:
            for p in self.bert_layer.parameters():
                p.requires_grad = False

        #Classification layer
        self.cls_layer = nn.Linear(768, 1)

    def forward(self, seq, attn_masks):
        '''
        Inputs:
            -seq : Tensor of shape [B, T] containing token ids of sequences
            -attn_masks : Tensor of shape [B, T] containing attention masks to be used to avoid contibution of PAD tokens
        '''

        #Feeding the input to BERT model to obtain contextualized representations
        cont_reps, _ = self.bert_layer(seq, attention_mask = attn_masks)

        #Obtaining the representation of [CLS] head
        cls_rep = cont_reps[:, 0]

        #Feeding cls_rep to the classifier layer
        logits = self.cls_layer(cls_rep)

        return logits


from dataclasses import dataclass
import os
import sys
import time
from datetime import datetime
import tempfile
import shutil
import json
from fastapi import APIRouter, HTTPException, UploadFile, Response
from app.controllers.processors import image as PI

from app.core import logger
from app.utils import utilities as U
from app.utils import constants as C

router = APIRouter(prefix="/document")
# logger = logger.log_setup(cfg, "app")
# create a temporary directory for processing
temp_dirpath = tempfile.mkdtemp()

@router.post("/classify")
async def Classifer(file: UploadFile):
    """API route to Classifer Claims."""
    _response = {
        "Transaction_Info": {"Transaction_ID": "1234", "Received_TS": "", "Processed_TS": ""},
        "prob": 0.0,
        "is_hw": False,
    }
    # 1. Get the line items
    start_time = time.time()
    start_dt = datetime.now()
    trxId = start_dt.strftime(r"%Y%m%d%H%M%S%f")
    logger.info(f"{U.prepend_msg(trxId)}{C.HEADER_10} Received Request {C.HEADER_10}")
    try:
        # Begin process

        logger.info(f"{U.prepend_msg(trxId)} - Begin Processing Request -> {start_time}")

        file_data = await file.read()
        # write the file to temperory directory for processing
        upload_dir = os.path.join(temp_dirpath, trxId)
        os.makedirs(upload_dir, exist_ok=True)
        upload_filepath = os.path.join(upload_dir, file.filename)  # f'./uploads/{txn_id}_{file.filename}'

        with open(upload_filepath, "wb") as upload_file:
            upload_file.write(file_data)
        logger.debug(f"{U.set_log_prefix(trxid = trxId)} - File saved upload_filepath.")

        result = await PI.process_image(trxId=trxId, image_path=upload_filepath)
        end_time = time.time()
        logger.info(
            f"{U.prepend_msg(trxId)} - End Processing Request -> {end_time} - Time Taken -> {end_time - start_time}"
        )
        end_dt = datetime.now()
        _response["Transaction_Info"]["Transaction_ID"] = trxId
        _response["Transaction_Info"]["Received_TS"] = start_dt
        _response["Transaction_Info"]["Processed_TS"] = end_dt
        _response["Model_Version"] = result["Model_Version"]
        _response["prob"] = result["prob"]
        _response["is_hw"] = result["is_hw"]
    except Exception as e:
        logger.error(
            f"{U.prepend_msg(trxId)} - there's an error uploading file. Please try again!!!",
            exc_info=True,
        )
        error_message = {
            "Status": C.ERROR,
            "Error": "Error while Uploading file. TRY AGAIN!!",
            "Error-Message": str(e),
        }
        raise HTTPException(status_code=418, detail=f"Error - there's an error uploading file. Please try again!!!")
    logger.info(f"{U.prepend_msg(trxId)} - Response Request -> {_response}")
    # remove the temporary directory
    shutil.rmtree(temp_dirpath, ignore_errors=True)
    return Response(
        content=json.dumps(_response, indent=4, sort_keys=True, default=str),
        media_type="application/json",
    )


# implements Kafka topic consumer functionality

import os
import threading
from confluent_kafka import Consumer, OFFSET_BEGINNING
import json
from producer import proceed_to_deliver
import base64
import subprocess

UPDATE_CWD = "updater/"
STORAGE_PATH = "tmp/"
UPDATE_SCRIPT_NAME = "./update-and-restart-app.sh"
APP_PATH = "../app/"
VERIFIER_SEAL = 'verifier_seal'

def check_seal(payload:str):
    if payload.endswith(VERIFIER_SEAL):
        return True
    return False

def strip_verifier_seal(sealed_payload:str):
    payload = sealed_payload[:-len(VERIFIER_SEAL)]
    return payload

def execute_update(id, details):
    # print(f"[debug]===== EXECUTING UPDATE ====\nDetails: {details}")
    print(f"[info]===== EXECUTING UPDATE {id} ====")
    update_payload_b64 = details['blob']
    if details['update_file_encoding'] != 'base64':
        print('[error] unsupported blob encoding')
        return
    update_sealed_payload = base64.b64decode(update_payload_b64).decode()
    if check_seal(update_sealed_payload) is not True:
        print('[error] verifier seal is missing or invalid')
        return
    else:
        payload = strip_verifier_seal(update_sealed_payload).encode()
        update_payload = base64.b64decode(payload)
    try:
        with open(UPDATE_CWD+STORAGE_PATH+id, "wb") as f:
            f.write(update_payload)
        f.close()
        result = subprocess.call(['bash', '-c', f"{UPDATE_SCRIPT_NAME} {STORAGE_PATH+id} {APP_PATH}"], cwd=UPDATE_CWD)
        print(f"[info] update result code {result}")
    except Exception as e:
        print(f'[error] failed to execute update: {e}. cwd: {os.getcwd()}')

def handle_event(id: str, details: dict):
    # print(f"[debug] handling event {id}, {details}")
    print(f"[info] handling event {id}, {details['source']}->{details['deliver_to']}: {details['operation']}")
    delivery_required = False
    try:
        if details['operation'] == 'proceed_with_update':
            # it's a request from manager for an update
            # get the blob by its id
            details['deliver_to'] = 'storage'
            details['operation'] = 'get_blob'
            delivery_required = True

        elif details['operation'] == 'blob_content':
            # blob with an update arrived
            execute_update(id, details)
    except Exception as e:
        print(f"[error] failed to handle request: {e}")
    if delivery_required:
        proceed_to_deliver(id, details)

def consumer_job(args, config):
    # Create Consumer instance
    verifier_consumer = Consumer(config)

    # Set up a callback to handle the '--reset' flag.
    def reset_offset(verifier_consumer, partitions):
        if args.reset:
            for p in partitions:
                p.offset = OFFSET_BEGINNING
            verifier_consumer.assign(partitions)

    # Subscribe to topic
    topic = "updater"
    verifier_consumer.subscribe([topic], on_assign=reset_offset)

    # Poll for new messages from Kafka and print them.
    try:
        while True:
            msg = verifier_consumer.poll(1.0)
            if msg is None:
                # Initial message consumption may take up to
                # `session.timeout.ms` for the consumer group to
                # rebalance and start consuming
                # print("Waiting...")
                pass
            elif msg.error():
                print(f"[error] {msg.error()}")
            else:
                # Extract the (optional) key and value, and print.
                try:
                    id = msg.key().decode('utf-8')
                    details = json.loads(msg.value().decode('utf-8'))
                    # print(
                    #     f"[debug] consumed event from topic {topic}: key = {id} value = {details}")
                    handle_event(id, details)
                except Exception as e:
                    print(
                        f"Malformed event received from topic {topic}: {msg.value()}. {e}")
    except KeyboardInterrupt:
        pass
    finally:
        # Leave group and commit final offsets
        verifier_consumer.close()

def start_consumer(args, config):
    threading.Thread(target=lambda: consumer_job(args, config)).start()

if __name__ == '__main__':
    start_consumer(None)


import json
from glob import glob
import yaml

def get_problems_by_name(response_file):
    with open(response_file) as f:
        response = json.load(f)

    def get_name(place):
        return place.get("permalink", place.get("id"))

    name_to_problems = {
        get_name(place): place["districtingProblems"] for place in response
    }
    assert all(isinstance(key, str) for key in name_to_problems)
    return name_to_problems

def add_problems(name_to_problems):
    for filename in glob("./data/*.yml"):
        print(filename)
        with open(filename) as f:
            record = yaml.safe_load(f)
        problems = name_to_problems[record["id"]]
        record["districtingProblems"] = problems
        with open(filename, "w") as f:
            yaml.safe_dump(record, f)

if __name__ == "__main__":
    name_to_problems = get_problems_by_name("../districtr/assets/data/response.json")
    add_problems(name_to_problems)


import numpy as np
import pandas as pd
from sklearn.preprocessing import MinMaxScaler, LabelEncoder, OneHotEncoder
from scipy import stats
from sklearn.linear_model import LinearRegression, Ridge
from sklearn.preprocessing import PolynomialFeatures
from sklearn.pipeline import make_pipeline
from sklearn.metrics import mean_squared_error
from sklearn.linear_model import LogisticRegression
from sklearn.model_selection import cross_val_score
from sklearn.metrics import classification_report
from imblearn.over_sampling import RandomOverSampler

# Data Preprocessing
df_train_clf = pd.read_csv('Classification/train_data.csv')
df_test_clf = pd.read_csv('Classification/test_data.csv')

# Categorical encoding
def one_hot_encode(df: pd.DataFrame, f_name: str):
    ohe = OneHotEncoder(sparse=False).fit_transform(df[f_name].to_numpy().reshape(-1, 1))
    encoded = pd.DataFrame(ohe, columns=[f"{f_name}{i}" for i in range(ohe.shape[1])])
    return pd.concat([df.drop([f_name], axis=1), encoded], axis=1)

f = 'auto_bitrate_state'
ohe_train_clf = one_hot_encode(df_train_clf, f)
ohe_test_clf = one_hot_encode(df_test_clf, f)

def label_encode(df: pd.DataFrame, f_name: str):
    label = LabelEncoder().fit_transform(df[f_name])
    res = df.copy()
    res[f_name] = label
    return res

f = 'auto_fec_state'
prep_train_clf = label_encode(ohe_train_clf, f)
prep_test_clf = label_encode(ohe_test_clf, f)

def scale(df: pd.DataFrame):
    return pd.DataFrame(MinMaxScaler().fit_transform(df), columns=df.columns)

X_train_clf, X_test_clf = scale(prep_train_clf.drop(['stream_quality'], axis=1)), scale(
    prep_test_clf.drop(['stream_quality'], axis=1))
y_train_clf, y_test_clf = prep_train_clf['stream_quality'], prep_test_clf['stream_quality']

# Feature scaling
df_train_reg: pd.DataFrame = pd.read_csv('Regression/train_data.csv')
df_test_reg: pd.DataFrame = pd.read_csv('Regression/test_data.csv')
reg_train_sc = scale(df_train_reg)
reg_test_sc = scale(df_test_reg)
X_train_reg, X_test_reg = reg_train_sc.drop(['target'], axis=1), reg_test_sc.drop(['target'], axis=1)
y_train_reg, y_test_reg = reg_train_sc['target'], reg_test_sc['target']
y_train_clf, y_test_clf = y_train_clf.astype(int), y_test_clf.astype(int)

# Dimensionality reduction (exact reasoning why I did it this way can be seen in ipynb)
X_train_reg, X_test_reg = X_train_reg.drop(['rtt_std'], axis=1), X_test_reg.drop(['rtt_std'], axis=1)
X_train_clf, X_test_clf = X_train_clf.drop(['auto_bitrate_state0'], axis=1), X_test_clf.drop(['auto_bitrate_state0'],
                                                                                axis=1)

# Data visualization can be seen in the ipynb

# Regression model
model1 = LinearRegression()
model2 = Ridge()
model1.fit(X_train_reg, y_train_reg)
model2.fit(X_train_reg, y_train_reg)

y_pred_reg1 = model1.predict(X_test_reg)
y_pred_reg2 = model2.predict(X_test_reg)

# Finding best polynomial regression model
res = []

for i in range(2, 5):
    poly = make_pipeline(PolynomialFeatures(i), LinearRegression())
    poly.fit(X_train_reg, y_train_reg)
    y_pred_reg3 = poly.predict(X_test_reg)
    res.append(mean_squared_error(y_test_reg, y_pred_reg3))
print('----------------------------------------------------')
print(f'Results of Polynomial Regression: {res}')

# Logistic Regression model

logistic = LogisticRegression(penalty='l2', max_iter=120000)
logistic.fit(X_train_clf, y_train_clf)
y_pred_clf = logistic.predict(X_test_clf)

# Results evaluation
# Regression

model1 = LinearRegression()
model2 = Ridge()
model3 = make_pipeline(PolynomialFeatures(2), LinearRegression())

m1_cross = cross_val_score(model1, X_train_reg, y_train_reg, cv=5)
m2_cross = cross_val_score(model2, X_train_reg, y_train_reg, cv=5)
m3_cross = cross_val_score(model3, X_train_reg, y_train_reg, cv=5)

print('----------------------------------------------------')

print(f'Model 1 Cross Validation mean: {np.mean(m1_cross)}, std: {np.std(m1_cross)}')
print(f'Model 2 Cross Validation mean: {np.mean(m2_cross)}, std: {np.std(m2_cross)}')
print(f'Model 3 Cross Validation mean: {np.mean(m3_cross)}, std: {np.std(m3_cross)}')
print('----------------------------------------------------')
print(f'Model 1 (Linear) - MSE:{mean_squared_error(y_test_reg, y_pred_reg1)}')
print(f'Model 2 (Linear + Ridge) - MSE:{mean_squared_error(y_test_reg, y_pred_reg2)}')
print(f'Model 3 (Polynomial with degree {np.argmin(res) + 2}) - MSE:{res[np.argmin(res)]}')

# Classification

logistic = LogisticRegression(penalty='l2', max_iter=120000)

log_cross = cross_val_score(logistic, X_train_clf, y_train_clf, cv=5)
print('----------------------------------------------------')
print(f'Logistic Regression Cross Validation mean: {np.mean(log_cross)}, std: {np.std(log_cross)}')
print('----------------------------------------------------')
print(classification_report(y_test_clf, y_pred_clf))

# Outliers detection
# Finding outliers
X_train_clf_o = X_train_clf.copy()
X_train_clf_o['target'] = y_train_clf
z = np.abs(stats.zscore(X_train_clf))

X_train_clf_o = X_train_clf_o[(z < 3).all(axis=1)]
print('----------------------------------------------------')
print(f'Removed Outliers: {X_train_clf.shape[0] - X_train_clf_o.shape[0]}')

X_train_clf_o, y_train_clf_o = X_train_clf_o.drop('target', axis=1), X_train_clf_o['target']

smt = RandomOverSampler(sampling_strategy='minority')
X_res, y_res = smt.fit_resample(X_train_clf_o, y_train_clf_o)

logistic = LogisticRegression(penalty='l2', max_iter=120000)

logistic.fit(X_res, y_res)
print('----------------------------------------------------')
print(f'Number of classes: 0 - {sum(y_res == 0)}, 1 - {sum(y_res == 1)}')

log_cross = cross_val_score(logistic, X_res, y_res, cv=5)
print('----------------------------------------------------')
print(f'Logistic Regression with sampling Cross Validation mean: {np.mean(log_cross)}, std: {np.std(log_cross)}')
print('----------------------------------------------------')
y_pred = logistic.predict(X_test_clf)
print('After outliers detection')
print(classification_report(y_test_clf, y_pred))
print('Before outliers detection')
print(classification_report(y_test_clf, y_pred_clf))


from typing import List
import numpy as np
from advent2022 import utils

DAY = 9

def load_and_parse_data(day: int, test: bool = False) -> List[str]:
    data = utils.get_input(day, test)
    return [l.split(" ") for l in data]

def new_pos(h, t):
    if abs(h[0] - t[0]) > 1 or abs(h[1] - t[1]) > 1:
        for move_idx in range(2):
            if h[move_idx] == t[move_idx]:
                continue
            move = 1 if h[move_idx] > t[move_idx] else -1
            t[move_idx] += move
    return t

def solve_rope(data, tail_length):
    visited = np.zeros((10000, 10000))
    h, tail = [4, 0], [[4, 0] for l in range(tail_length)]
    visited[tail[0][0], tail[0][1]] = 1
    for l in data:
        direction, distance = l[0], int(l[1])
        if direction == "U":
            idx, step = 0, -1
        elif direction == "D":
            idx, step = 0, 1
        elif direction == "L":
            idx, step = 1, -1
        elif direction == "R":
            idx, step = 1, 1

        for i in range(distance):
            h[idx] += step
            for j, _ in enumerate(tail):
                follow = tail[j - 1] if j > 0 else h
                tail[j] = new_pos(follow, tail[j])
                if j == tail_length - 1:
                    visited[tail[j][0], tail[j][1]] = 1

    return np.sum(visited)

def solve_part_1(data):
    return solve_rope(data, 1)

def solve_part_2(data):
    return solve_rope(data, 9)


import os
from tqdm import tqdm
import datetime
import pandas as pd
import logging

logging.basicConfig(format='%(asctime)s - %(levelname)s - %(message)s',
                    level=logging.INFO,
                    # datefmt='%d-%b-%y %H:%M:%S'
                    )

def get_date_df(file_folder, days):
    process_bar = tqdm(os.listdir(file_folder))
    date_list = []
    name_list = []
    for file_name in process_bar:
        process_bar.set_description("Processing {}".format(file_name))
        paper_date = file_name[:10]
        paper_name = file_name
        date_list.append(paper_date)
        name_list.append(paper_name)
    dates_list = [date_list, name_list]
    df = pd.DataFrame(dates_list)
    df = df.transpose()
    df.columns = ['date', 'file_name']
    df['date'] = pd.to_datetime(df['date'])
    df.sort_values(by=['date'], inplace=True)
    first_date = df.iloc[0]['date']
    first_year = first_date.year
    new_year = pd.Timestamp(f'{first_year}-01-01T00')
    if first_date == new_year:
        logging.info(f"The first day of year {first_year} already exists!")
    else:
        logging.info(f"Creating placeholder data for the first day of year {first_year}")
        df.loc[-1] = [new_year, 'placeholder.txt']
        df.index = df.index + 1
        df = df.sort_index()
    t = df.groupby(pd.Grouper(key="date", axis=0, freq=days, sort=True))['file_name'].apply(list).reset_index(
        name='file_name_list')
    t.to_csv("sorted_df.csv")
    return t

def creat_data(sorted_df, output_folder, file_folder):
    if not os.path.exists(output_folder):
        os.makedirs(output_folder)
    for idx, row in tqdm(sorted_df.iterrows()):
        if row['file_name_list']:
            date = row['date'].date()
            file_name_list = row['file_name_list']
            if not file_name_list[0] == 'placeholder.txt':
                text_str = []
                for file_name in file_name_list:
                    file_path = os.path.join(file_folder, file_name)
                    with open(file_path, 'r', encoding='utf-8') as infile:
                        for line in infile:
                            text_str.append(line)
                out_file_path = os.path.join(output_folder, f"{date}.txt")
                with open(out_file_path, 'w', encoding='utf-8') as out_file:
                    out_file.write("".join(text_str))

def main():
    # adjust the following variable values to get the desired output
    file_folder = "output_data"
    days = "10D"
    output_folder = "data_by_10D"
    sorted_df = get_date_df(file_folder, days)
    creat_data(sorted_df, output_folder, file_folder)

if __name__ == '__main__':
    main()


from django.shortcuts import render, get_object_or_404
from zoo.models import Category, Product
from django.views.generic import ListView
from django.core.paginator import Paginator
from zoo.forms import ZooSearchForm

class CategoryListView(ListView):
    model = Product
    template_name = 'catalog.html'
    context_object_name = 'products'

    paginate_by = 12

    def get_context_data(self, *, object_list=None, **kwargs):
        context = super().get_context_data(object_list=object_list, **kwargs)
        context['object-count'] = self.model.objects.count()
        paginator = Paginator(self.model.object_list, self.paginate_by)
        try:
            page = self.request.GET.get('page')
        except:
            page = 1

        try:
            context[self.context_object_name] = paginator.page(page)
        except:
            context[self.context_object_name] = paginator.page(1)

        context['object-count'] = self.model.objects.count()
        context['paginator'] = paginator
        return context

    def get_context_data(self,*,object_list=None, **kwargs):
         context = super().get_context_data(object_list=object_list, **kwargs)

         form = ZooSearchForm(self.request.GET)
         if form.is_valid():
             pass

         else:
             form = ZooSearchForm()

         context['form'] = form
         return context

    def get(self, request, *args, **kwargs):
        form = ZooSearchForm(self.request.GET)
        if form.is_valid():
            cd = form.cleaned_data
            product = self.model.objects.filter(name__icontains=cd['search'])
        else:
            product = self.model.objects.all()
        return render(request, self.template_name, self.get_context_data(object_list=product))

class ZooView (ListView):
   model = Product
   context_object_name = 'product'
   template_name = 'main.html'

class Zoo2View (ListView):
   model = Category
   context_object_name = 'category'
   template_name = 'tov.html'

def uslygi_view(request):
    context = {'page':'uslygi'}
    return render(request, 'uslygi.html',context)


import unittest
import importlib
import mock
import datetime
from oslo_utils import timeutils

evacuate_lbaas = importlib.import_module("neutron-evacuate-lbaasv2-agent")

class FakeSqlResult():
    def fetchall(self):
        return [
            ['1', 'healthy', timeutils.utcnow()],
            ['2', 'dead', timeutils.utcnow() - datetime.timedelta(seconds=100)]
        ]

class FakeSqlConnection():
    def execute(self, *args):
        return FakeSqlResult()

class TestEvacuateLbaasV2Agents(unittest.TestCase):
    def setUp(self):
        self.evacuate_lbaas = evacuate_lbaas.EvacuateLbaasV2Agent()
        self.evacuate_lbaas.connection = FakeSqlConnection()
        self.evacuate_lbaas.host_to_evacuate = "evacuateme"

    def test_available_agents_exclude_dead_agents(self):
        self.assertEqual(
            [{'host': 'healthy', 'id': '1'}],
            self.evacuate_lbaas.available_destination_agents()
        )

    def test_reassing_single_lb_returns_one_agent(self):
        agents = [
            {'host': 'node1', 'id': '1'},
            {'host': 'node2', 'id': '2'},
            {'host': 'node3', 'id': '3'}
        ]
        loadbalancers = ['abc']

        res = self.evacuate_lbaas.reassign_loadbalancers(loadbalancers, agents)
        self.assertEqual(1, len(res))

    @mock.patch('neutron-evacuate-lbaasv2-agent.'
                'EvacuateLbaasV2Agent.loadbalancers_on_agent')
    @mock.patch('neutron-evacuate-lbaasv2-agent.'
                'RemoteLbaasV2Cleanup')
    def test_restarts_agents_using_crm_on_ha(self, mock_cleanup, mock_lbaas):
        mock_lbaas.return_value = ['lb1']
        evacuate_lbaas.cfg.CONF.set_override("use_crm", True)

        self.evacuate_lbaas.run()
        self.assertEqual(
            mock_cleanup.return_value.restart_lbaasv2_agent_crm.call_count,
            2
        )
        self.assertEqual(
            mock_cleanup.return_value.restart_lbaasv2_agent_systemd.call_count,
            0
        )

    @mock.patch('neutron-evacuate-lbaasv2-agent.'
                'EvacuateLbaasV2Agent.loadbalancers_on_agent')
    @mock.patch('neutron-evacuate-lbaasv2-agent.'
                'RemoteLbaasV2Cleanup')
    def test_restarts_agents_using_systemd_no_ha(self,
                                                 mock_cleanup,
                                                 mock_lbaas):
        mock_lbaas.return_value = ['lb1']
        evacuate_lbaas.cfg.CONF.set_override("use_crm", False)
        self.evacuate_lbaas.run()
        self.assertEqual(
            mock_cleanup.return_value.restart_lbaasv2_agent_crm.call_count,
            0
        )
        self.assertEqual(
            mock_cleanup.return_value.restart_lbaasv2_agent_systemd.call_count,
            2
        )


"""Engi Init

Walks through the setup enabling a user to work with ENGI.

Usage:
  engi init
  engi (-h | --help)
  engi --version

Options:
  -h --help                Show this screen
"""
from git import Repo
from gitsecrets import GitSecrets
import gnupg
import os
import re
import sys
from pathlib import Path
import subprocess
import tempfile
import urllib.request
from yaml import load, dump

from InquirerPy import inquirer
from InquirerPy.utils import color_print
from InquirerPy.validator import PathValidator

try:
    from yaml import CLoader as Loader, CDumper as Dumper
except ImportError:
    from yaml import Loader, Dumper

home_path = Path('.')
DOCKER_TEMPLATES='https://raw.githubusercontent.com/engi-network/docker-templates/master/'

gpg = gnupg.GPG()

ENGI_DIR = Path(".engi")
ENGI_YAML_FILE_NAME = "docker-compose.yml"
ENGI_YAML_FILE = ENGI_DIR.joinpath(ENGI_YAML_FILE_NAME)
ENGI_DOCKER_FILE_NAME = "tests.Dockerfile"
ENGI_DOCKER_FILE = ENGI_DIR.joinpath(ENGI_DOCKER_FILE_NAME)
ENGI_DOCKER_IGNORE_FILE_NAME = "tests.Dockerfile.dockerignore"
ENGI_DOCKER_IGNORE_FILE = ENGI_DIR.joinpath(ENGI_DOCKER_IGNORE_FILE_NAME)

def is_valid_email_address(email_address):
   match = re.fullmatch(r"[^@]+@[^@]+\.[^@]+", email_address)
   return bool(match)

def download_framework_dockerfiles(framework):
    dockerfile = f'{DOCKER_TEMPLATES}/{framework}/{ENGI_DOCKER_FILE_NAME}'
    compose = f'{DOCKER_TEMPLATES}/{framework}/{ENGI_YAML_FILE_NAME}'
    dockerignore = f'{DOCKER_TEMPLATES}/{framework}/{ENGI_DOCKER_IGNORE_FILE_NAME}'

    ENGI_DIR.mkdir(parents=True, exist_ok=True)

    urllib.request.urlretrieve(dockerfile, ENGI_DOCKER_FILE)
    urllib.request.urlretrieve(compose, ENGI_YAML_FILE)
    urllib.request.urlretrieve(dockerignore, ENGI_DOCKER_IGNORE_FILE)

def setup_git_secrets():
    with tempfile.NamedTemporaryFile() as tmp:
        download = f'{DOCKER_TEMPLATES}/engi-pubkey'
        urllib.request.urlretrieve(download, tmp.name)
        gpg.import_keys_file(tmp.name)

    pubkey_id = inquirer.text(
        message="Git secret pubkey id (email):",
        validate=is_valid_email_address,
        invalid_message="Please enter valid email"
    ).execute()

    try:
        subprocess.run(['git-secret', 'init'])
        subprocess.run(['git-secret', 'tell', pubkey_id])
        subprocess.run(['git-secret', 'tell', 'circleci@engi.network'])
    except FileNotFoundError:
        color_print(formatted_text=[("class:aa", "git-secret doesn't seem to be installed")], style={"aa": "red"})
        sys.exit()

def main():
    framework = inquirer.select(
        message="Select the test output format to parse",
        choices=["pytest", "jest", "dotnet"],
    ).execute()

    if framework == 'jest' and inquirer.confirm(message="Are you using design tests?").execute():
        framework = "jest-storybook"

    # TODO: handling pre-existing containerization code (patch, guide)
    print('Preparing to generate Docker files...')
    if os.path.exists('./Dockerfile') or os.path.exists('./docker-compose.yml'):
        print('You already have Docker configured. You need to update your Dockerfile and docker-compose.yml manually. You Skipping this step.')
    else:
        print('Downloading containerization files for and placing them in .engi directory', framework)
        download_framework_dockerfiles(framework)

    #tests = inquirer.filepath(
    #    message="Test file path:",
    #    default=home_path,
    #    validate=PathValidator(is_dir=True, message="Filepath"),
    #).execute()

    secret = inquirer.confirm(message="Do you want to set up a secret store (e.g. for hiding test data)?").execute()
    if secret:
        setup_git_secrets()

if __name__ == "__main__":
    main()


from django.db.models.signals import pre_save

class Profile(models.Model):
    to_receive_new_user = models.BooleanField(default=False)
    created_at = models.DateTimeField(auto_now_add=True)

    @receiver(pre_save, sender=User)
    def update_profile(sender, instance, **kwargs):
    	instance.to_receive_new_user = True
    	instance.save()
    	return instance


from minio import Minio
import redis
import time
import json

APP = "wc"
minio_client = Minio("minio-service.yasb-mapreduce-db.svc.cluster.local:9000", access_key="admin123", secret_key="admin123", secure=False)
redis_client = redis.Redis(host="redis.yasb-mapreduce-db.svc.cluster.local", port=6379)

def handle(req):
    """handle a request to the function
    Args:
        req (str): request body
    """
    event = json.loads(req)

    input_name = event["input_name"]
    input_part = int(event["input_part"])
    reduce_num = int(event["reduce_num"])

    read_start = time.time()

    input_object = minio_client.get_object(input_name, "part-%d" % input_part)
    input_data = input_object.data.decode("utf-8")
    print("------------------------------------------------------input_data_size%d" % len(input_data))
    read_end = time.time()

    counts = {}

    lines = input_data.split("\n")

    print("------------------------------------------------------line_size%d" % len(lines))
    for line in lines:
        words = line.strip().split(" ")
        for word in words:
            if word.isalpha():
                if word not in counts:
                    counts[word] = 0
                counts[word] = counts[word] + 1
    print("------------------------------------------------------count_size%d" % len(counts))
    shuffle = {}
    for i in range(reduce_num):
        shuffle[i] = ''

    for word, count in counts.items():
        reduce_id = hash(word) % reduce_num
        shuffle[reduce_id] = shuffle[reduce_id] + "%s:%d;" % (word, count)

    for i in range(reduce_num):
        if shuffle[i][-1] == ";":
            shuffle[i] = shuffle[i][:-1]

    com_end = time.time()

    for i in range(reduce_num):
        if not shuffle[i] == '':
            name = "%s:%s:%d:%d" % (input_name, APP, input_part, i)
            redis_client.set(name, shuffle[i])

    store_end = time.time()

    result = {
        "read_start": read_start,
        "read_end": read_end,
        "com_end": com_end,
        "store_end": store_end
    }

    return json.dumps(result)


from rest_framework import routers
from .viewsets import CustomUserViewSet, AccountViewset
from django.urls import path
from  . import views

urlpatterns = [
    path( 'user/login', views.login ),
    path( 'user/register', views.Register ),
    path( 'account/getAccount', views.getAccountByEmail ),
    path( 'account/addFounds', views.addFounds ),
    path( 'account/takeOutAmount', views.takeOutAmount )
]

route = routers.SimpleRouter()
route.register( 'account', AccountViewset )
route.register( 'user', CustomUserViewSet )
urlpatterns += route.urls


import re
import sys
import datetime
from itertools import zip_longest
from email.utils import getaddresses, parsedate_to_datetime
from email.header import decode_header, Header
from typing import AnyStr, Union, Optional, Tuple, Iterable, Any, List, Dict, Iterator

from .consts import SHORT_MONTH_NAMES, MailMessageFlags
from . import imap_utf7

def clean_uids(uid_set: Union[str, Iterable[str]]) -> str:
    """
    Prepare set of uid for use in IMAP commands
    uid RE patterns are not strict and allow invalid combinations, but simple. Example: 2,4:7,9,12:*
    :param uid_set:
        str, that is comma separated uids
        Iterable, that contains str uids
    :return: str - uids, concatenated by a comma
    """
    # str
    if type(uid_set) is str:
        if re.search(r'^([\d*:]+,)*[\d*:]+$', uid_set):  # *optimization for already good str
            return uid_set
        uid_set = uid_set.split(',')
    # check uid types
    for uid in uid_set:
        if type(uid) is not str:
            raise TypeError('uid "{}" is not string'.format(str(uid)))
        if not re.match(r'^[\d*:]+$', uid.strip()):
            raise TypeError('Wrong uid: "{}"'.format(uid))
    return ','.join(i.strip() for i in uid_set)

def check_command_status(command_result: tuple, exception: type, expected='OK'):
    """
    Check that IMAP command responses status equals <expected> status
    If not, raise specified <exception>
    :param command_result: imap command result: tuple(typ, data)
    :param exception: exception subclass of UnexpectedCommandStatusError, that raises
    :param expected: expected command status
    """
    typ, data = command_result[0], command_result[1]
    if typ != expected:
        raise exception(command_result=command_result, expected=expected)

def decode_value(value: AnyStr, encoding: Optional[str] = None) -> str:
    """Converts value to utf-8 encoding"""
    if isinstance(encoding, str):
        encoding = encoding.lower()
    if isinstance(value, bytes):
        try:
            return value.decode(encoding or 'utf-8', 'ignore')
        except LookupError:  # unknown encoding
            return value.decode('utf-8', 'ignore')
    return value

class EmailAddress:
    """Parsed email address info"""
    __slots__ = 'name', 'email'

    def __init__(self, name: str, email: str):
        self.name = name
        self.email = email

    @property
    def full(self):
        return '{} <{}>'.format(self.name, self.email) if self.name and self.email else self.name or self.email

    def __repr__(self):
        return "{}(name={}, email={})".format(
            self.__class__.__name__, repr(self.name), repr(self.email))

    def __eq__(self, other):
        return all(getattr(self, i) == getattr(other, i) for i in self.__slots__)

def parse_email_addresses(raw_header: Union[str, Header]) -> Tuple[EmailAddress, ...]:
    """
    Parse email addresses from header
    :param raw_header: example: '=?UTF-8?B?0J7Qu9C1=?= <name@company.ru>,\r\n "\'\\"z, z\\"\'" <imap.tools@ya.ru>'
    :return: (EmailAddress, ...)
    """
    result = []
    if type(raw_header) is Header:
        raw_header = decode_value(*decode_header(raw_header)[0])
    for raw_name, email in getaddresses([raw_header.replace('\r\n', '').replace('\n', '')]):
        name = decode_value(*decode_header(raw_name)[0]).strip()
        email = email.strip()
        if not (name or email):
            continue
        result.append(EmailAddress(
            name=name or (email if '@' not in email else ''),
            email=email if '@' in email else '',
        ))
    return tuple(result)

def parse_email_date(value: str) -> datetime.datetime:
    """
    Parsing the date described in rfc2822
    Result datetime may be naive or with tzinfo
    1900-1-1 for unparsed
    """
    try:
        return parsedate_to_datetime(value)
    except Exception:  # noqa
        pass
    match = re.search(
        r'(?P<date>\d{1,2}\s+(' + '|'.join(SHORT_MONTH_NAMES) + r')\s+\d{4})\s+' +
        r'(?P<time>\d{1,2}:\d{1,2}(:\d{1,2})?)\s*' +
        r'(?P<zone_sign>[+-])?(?P<zone>\d{4})?',
        value
    )
    if match:
        group = match.groupdict()
        day, month, year = group['date'].split()
        time_values = group['time'].split(':')
        zone_sign = int('{}1'.format(group.get('zone_sign') or '+'))
        zone = group['zone']
        try:
            return datetime.datetime(
                year=int(year),
                month=SHORT_MONTH_NAMES.index(month) + 1,
                day=int(day),
                hour=int(time_values[0]),
                minute=int(time_values[1]),
                second=int(time_values[2]) if len(time_values) > 2 else 0,
                tzinfo=datetime.timezone(datetime.timedelta(
                    hours=int(zone[:2]) * zone_sign,
                    minutes=int(zone[2:]) * zone_sign
                )) if zone else None,
            )
        except ValueError:
            pass
    return datetime.datetime(1900, 1, 1)

def quote(value: AnyStr) -> AnyStr:
    if isinstance(value, str):
        return '"' + value.replace('\\', '\\\\').replace('"', '\\"') + '"'
    else:
        return b'"' + value.replace(b'\\', b'\\\\').replace(b'"', b'\\"') + b'"'

def pairs_to_dict(items: List[Any]) -> Dict[Any, Any]:
    """Example: ['MESSAGES', '3', 'UIDNEXT', '4'] -> {'MESSAGES': '3', 'UIDNEXT': '4'}"""
    if len(items) % 2 != 0:
        raise ValueError('An even-length array is expected')
    return dict((items[i * 2], items[i * 2 + 1]) for i in range(len(items) // 2))

def chunks(iterable: Iterable[Any], n: int, fill_value: Optional[Any] = None) -> Iterator[Tuple[Any, ...]]:
    """
    Group data into fixed-length chunks or blocks
        [iter(iterable)]*n creates one iterator, repeated n times in the list
        izip_longest then effectively performs a round-robin of "each" (same) iterator
    Examples:
        chunks('ABCDEFGH', 3, '?') --> [('A', 'B', 'C'), ('D', 'E', 'F'), ('G', 'H', '?')]
        chunks([1, 2, 3, 4, 5], 2) --> [(1, 2), (3, 4), (5, None)]
    """
    return zip_longest(*[iter(iterable)] * n, fillvalue=fill_value)

def encode_folder(folder: AnyStr) -> bytes:
    """Encode folder name"""
    if isinstance(folder, bytes):
        return folder
    else:
        return quote(imap_utf7.encode(folder))

def clean_flags(flag_set: Union[str, Iterable[str]]) -> List[str]:
    """
    Check the correctness of the flags
    :return: list of str - flags
    """
    if type(flag_set) is str:
        flag_set = [flag_set]
    upper_sys_flags = tuple(i.upper() for i in MailMessageFlags.all)
    for flag in flag_set:
        if not type(flag) is str:
            raise ValueError('Flag - str value expected, but {} received'.format(type(flag_set)))
        if flag.upper() not in upper_sys_flags and flag.startswith('\\'):
            raise ValueError('Non system flag must not start with "\\"')
    return flag_set

def check_timeout_arg_support(timeout):
    """If timeout arg not supports - raise ValueError"""
    if timeout is not None and sys.version_info.minor < 9:
        raise ValueError('imaplib.IMAP4 timeout argument supported since python 3.9')

def replace_html_ct_charset(html: str, new_charset: str) -> str:
    """Replace charset in META tag with content-type attribute in HTML text"""
    meta_ct_match = re.search(r'<\s*meta .*?content-type.*?>', html, re.IGNORECASE | re.DOTALL)
    if meta_ct_match:
        meta = meta_ct_match.group(0)
        meta_new = re.sub(r'charset\s*=\s*[a-zA-Z0-9_:.+-]+', 'charset={}'.format(new_charset), meta, 1, re.IGNORECASE)
        html = html.replace(meta, meta_new)
    return html


#pip install opencv-python
import cv2

webcam = cv2.VideoCapture(0)

try:
    if webcam.isOpened():
        validacao, frame = webcam.read()
        cv2.imwrite("testeWebcam.png", frame)
    webcam.release()
    cv2.destroyAllWindows()
except:
    print("Não foi possível abrir a câmera.")


import os
import torch
import torchvision.transforms as transforms
import torchvision.datasets as datasets
from .. import config as conf

processed_data_dir = conf.processed_data_dir
raw_data_dir       = conf.raw_data_dir

# Create the directories if they don't exist
os.makedirs(raw_data_dir, exist_ok=True)
os.makedirs(processed_data_dir, exist_ok=True)

# Define the normalization transform
normalize = transforms.Compose([
    transforms.ToTensor(),
    transforms.Normalize((0.1307,), (0.3081,))  # MNIST specific mean and std
])

# Download and normalize the MNIST dataset
train_dataset = datasets.MNIST(raw_data_dir, train=True, download=True, transform=normalize)
test_dataset = datasets.MNIST(raw_data_dir, train=False, download=True, transform=normalize)

# Split the original training data into a new training set and a validation set
train_size = int(0.8 * len(train_dataset))  # 80% of the data for training
val_size = len(train_dataset) - train_size  # 20% of the data for validation
train_dataset, val_dataset = torch.utils.data.random_split(train_dataset, [train_size, val_size])

# Save the datasets
torch.save(train_dataset, os.path.join(processed_data_dir, "train_normalized.pt"))
torch.save(val_dataset, os.path.join(processed_data_dir, "val_normalized.pt"))
torch.save(test_dataset, os.path.join(processed_data_dir, "test_normalized.pt"))


# Python 3.5.1 tested
#
# If downloaded set is out of date, ie. does not contain most recent comic
# then the comics since most recently downloaded will be downloaded.
# If however the most recent comic is present, the script will scan all
# previous comics, ensuring ALL of them are present
#
# It is possible that script will need to run TWICE to ensure ALL comics
# are present, though this still maintains average case having optimal performance
#
# Script requires no user interaction from being started to self-termination

from os import listdir, makedirs
from os.path import exists, isfile, join
import requests as web
import urllib.request as image_downloader
from bs4 import BeautifulSoup

baseurl = 'http://xkcd.com/'
direc = 'images/'

def find_max_already_downloaded():
    only_files = [f for f in listdir(direc) if isfile(join(direc, f))]
    max_file_num = 0
    for file in only_files:
        n = int(file.split(' ')[0])
        if n > max_file_num:
            max_file_num = n
    return max_file_num

def detect_max_page():
    max_num_found = 1
    plain_text = web.get(baseurl).text
    soup = BeautifulSoup(plain_text, 'html.parser')
    for a in soup.findAll('a', {}):
        link_no_slashes = a.get('href').replace('/', '')
        if link_no_slashes.isdigit():
            n = int(link_no_slashes)
            if n > max_num_found:
                max_num_found = n
    return max_num_found + 1 # plus 1 because link found is to previous comic

def img_spider(start_page, max_page):
    if start_page > max_page:
        print("Unusual start_page max_page combination...")
        print("Reverting to scanning all pages, starting at 1")
        start_page = 1

    page = start_page
    while page <= max_page:
        url = baseurl + str(page) + '/'
        plain_text = web.get(url).text
        soup = BeautifulSoup(plain_text, 'html.parser')
        for img in soup.findAll('img', {}):
            img_url = img.get('src')
            if 'comics' in img_url:
                filename = img_url.split('/')[-1].replace('_(1)', '')
                filepath = direc + '{} - '.format(page) + filename
                if exists(filepath):
                    print('{}/{} - Exists already - '.format(page, max_page) + filename)
                else:
                    print('{}/{} - Downloading... - '.format(page, max_page) + filename)
                    image_downloader.urlretrieve('http:' + img_url, filepath)
                break # to skip all imgs past the comic img, as we know we don't care
        page += 1

def scan_unseen_pages():
    if not exists(direc):
        makedirs(direc)

    start_page = 1 + find_max_already_downloaded()
    end_page = detect_max_page()

    print('Scanning from {} to {}...'.format(start_page, end_page))
    img_spider(start_page, end_page)
    print('Scan completed')

scan_unseen_pages()


# coding=utf-8
import xml.etree.ElementTree as ET
import matplotlib.pyplot as plt
import collections
import pprint as pp
from scipy.stats import gaussian_kde
from numpy import arange

def violin_plot(ax,data,pos, bp=False):
	'''
	create violin plots on an axis
	'''
	dist = max(pos)-min(pos)
	w = min(0.15*max(dist,1.0),0.5)
	for d,p in zip(data,pos):
		k = gaussian_kde(d) #calculates the kernel density
		m = k.dataset.min() #lower bound of violin
		M = k.dataset.max() #upper bound of violin
		x = arange(m,M,(M-m)/100.) # support for violin
		v = k.evaluate(x) #violin profile (density curve)
		v = v/v.max()*w #scaling the violin to the available space
		ax.fill_betweenx(x,p,v+p,facecolor='y',alpha=0.3)
		ax.fill_betweenx(x,p,-v+p,facecolor='y',alpha=0.3)
	if bp:
		ax.boxplot(data,notch=1,positions=pos,vert=1)

plotati = True

mjerenjaPoSatima = dict()
#odabirSataDat = open("./Stats/ukupniStats.txt", "w")
for sat in range(24): #za svaki sat
	#statsFile = open("./Stats/stats"+str(sat)+".txt", "w")
	svaMjerenja = dict()
	for j in range(1, 9): #za svake postavke
		velPop = int(ET.parse("./"+str(j)+"/Config.xml").find("Algorithm").find("PopulationSize").text)
		# a = int(ET.parse("./"+str(j)+"/Config.xml").find(".//Algorithm/PopulationSize").text)
		# b = int(ET.parse("./"+str(j)+"/Config.xml").find(".//Algorithm/Termination/Entry[@name='NumberOfGenerations']").text)
		#print a, b, a*b
		vrijednosti = []
		for batchNode in ET.parse("./"+str(j)+"/Logovi/log"+str(sat)+".txt").getroot().findall("Batch"):
			jedinka = batchNode.findall("Jedinka")[-1]
			vrijednosti.append(float(jedinka.get("greska")))
		svaMjerenja[velPop] = vrijednosti

	ukupnaGreska = sum([sum(m) for m in svaMjerenja.values()])
	brojGreski = sum([len(m) for m in svaMjerenja.values()])
	srednjaGreska = ukupnaGreska/brojGreski
	for m in svaMjerenja.values():
		for i in range(len(m)):
			m[i] /= srednjaGreska

	mjerenjaPoSatima[sat] = svaMjerenja
#mjerenjaPoSatima[sat][velPop][batchNo]

mjerenjaPoVelPop = dict()
for sat in mjerenjaPoSatima:
	for vp in mjerenjaPoSatima[sat]:
		if vp not in mjerenjaPoVelPop:
			mjerenjaPoVelPop[vp] = []
		mjerenjaPoVelPop[vp].extend(mjerenjaPoSatima[sat][vp])
#mjerenjaPoVelPop[velPop]

plt.figure()
plt.xlabel('Velicina Populacije')
plt.ylabel('Normirana Greska Jedinke')
plt.title("Broj evaluacija = 50 000")
x = sorted(mjerenjaPoVelPop.keys())
y = [mjerenjaPoVelPop[i] for i in x]
plt.xticks(x,x)
plt.boxplot(y, 0, '')

plt.figure()
plt.xlabel('Velicina Populacije')
plt.ylabel('Normirana Greska Jedinke')
plt.title("Broj evaluacija = 50 000")
plt.xticks(x,x)
plt.boxplot(y)
plt.show()


import os
from django.conf import settings
from django.shortcuts import render, redirect
import face_recognition
import numpy as np
import cv2
from os.path import dirname, join
from django.apps import apps
from django.core.files.storage import FileSystemStorage
from django.contrib.auth.views import LoginView
from django.contrib.auth import get_user_model
from django.contrib.auth import login

class InstructorLogin(LoginView):

    template_name = 'Instructor/login.html'

    def post(self, request):
        username = request.POST['username']
        User = get_user_model()
        user = User.objects.get(username=username)

        if user.is_instructor==1:
            login(request, user)

            return redirect('Instructor-Home')
        else:
            return render(request,'Admin/FrontPage.html')

def findEncondings(images):

    encodeList = []
    for img in images:
        img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
        encode = face_recognition.face_encodings(img)[0]
        encodeList.append(encode)
    return encodeList

def home(request):
    c_user = apps.get_model('Admin', 'CustomUser')
    Profile = apps.get_model('Admin', 'Profile')
    course = apps.get_model('Admin', 'course')
    registration = apps.get_model('Admin', 'registration')
    attendance = apps.get_model('Admin', 'attendance')
    instructor = request.user
    inst_course = course.objects.get(instructor=instructor)
    students = inst_course.registration.all()
    # path = 'C:/Users/mg/PycharmProjects/testattendence/AttendanceSystem'
    path = settings.BASE_DIR
    images = []
    classNames = []
    students_present = []
    for student in students:
        s_profile = Profile.objects.get(user=student)
        curImg = cv2.imread(f'{path}/media/{s_profile.image}')
        images.append(curImg)
        classNames.append(student.username)
    print(classNames)

    if request.method == 'POST':
        myfile = request.FILES['document']
        print(myfile.name)
        fs = FileSystemStorage()
        imgname = fs.save(myfile.name, myfile)
        url = fs.url(imgname)
        print(url)
        print(imgname)
        date=request.POST.get('Date')
        print(date)
        encodeListKnown = findEncondings(images)
        print('Encoding complete')
        # encoding end

        faces = []
        prototxtPath = join(dirname(__file__), "deploy.prototxt.txt")
        modelPath = join(dirname(__file__), "res10_300x300_ssd_iter_140000.caffemodel")
        # image path
        # imagePath = 'C:/Users/mg/Desktop/DSC_0227.jpg'
        confidence1 = 0.1
        # construct the argument parse and parse the arguments
        # load our serialized model from disk
        print("[INFO] loading model...")
        net = cv2.dnn.readNetFromCaffe(prototxtPath, modelPath)
        # load the input image and construct an input blob for the image
        # by resizing to a fixed 300x300 pixels and then normalizing it
        image = cv2.imread(f'{path}{url}')
        fs.delete(imgname)
        image = cv2.resize(image, (0, 0), None, 0.25, 0.25)
        (h, w) = image.shape[:2]
        blob = cv2.dnn.blobFromImage(cv2.resize(image, (300, 300)), 1.0,
                                     (300, 300), (104.0, 177.0, 123.0))
        # pass the blob through the network and obtain the detections and
        # predictions
        print("[INFO] computing object detections...")
        net.setInput(blob)
        detections = net.forward()
        # loop over the detections
        for i in range(0, detections.shape[2]):
            confidence = detections[0, 0, i, 2]
            if confidence > 0.7:
                box = detections[0, 0, i, 3:7] * np.array([w, h, w, h])
                (startX, startY, endX, endY) = box.astype("int")
                faces.append((startY, endX, endY, startX))
        encodesCurFrame = face_recognition.face_encodings(image, faces)
        print("encode")
        for encodeFace, faceLoc in zip(encodesCurFrame, faces):
            matches = face_recognition.compare_faces(encodeListKnown, encodeFace)
            faceDis = face_recognition.face_distance(encodeListKnown, encodeFace)
            matchIndex = np.argmin(faceDis)

            if matches[matchIndex]:
                name = classNames[matchIndex]
                students_present.append(name)
                print(name)
                y1, x2, y2, x1 = faceLoc
                cv2.rectangle(image, (x1, y1), (x2, y2), (0, 255, 0), 2)
                cv2.rectangle(image, (x1, y2 - 35), (x2, y2), (0, 255, 0))
                cv2.putText(image, name, (x1 + 6, y2 - 6), cv2.FONT_HERSHEY_COMPLEX, 1, (0, 0, 255), 2)
                # markAttendence(name)
            else:
                y1, x2, y2, x1 = faceLoc
                cv2.rectangle(image, (x1, y1), (x2, y2), (0, 255, 0), 2)
                cv2.rectangle(image, (x1, y2 - 35), (x2, y2), (0, 255, 0))
                cv2.putText(image, "unknown", (x1 + 6, y2 - 6), cv2.FONT_HERSHEY_COMPLEX, 1, (0, 0, 255), 2)

        # show the output image
        # cv2.imshow("Output", image)
        # cv2.waitKey(0)
        print(students_present)
        students_present = set(students_present)
        print(students_present)
        marked = []
        for s in students:
            if s.username in students_present:
                stu = c_user.objects.get(username=s)
                obj = attendance()
                obj.student = stu
                obj.course = inst_course
                obj.status = 'p'
                obj.date = request.POST.get('Date')
                obj.save()
                marked.append(obj)
            else:
                stu = c_user.objects.get(username=s)
                obj = attendance()
                obj.student = stu
                obj.course = inst_course
                obj.status = 'A'
                obj.date = request.POST.get('Date')
                obj.save()
                marked.append(obj)
        data = {"marked": marked}
        print(marked[0].course)
        return render(request, 'Instructor/home.html', data)
    else:
        return render(request, 'Instructor/home.html')


from http.server import HTTPServer, BaseHTTPRequestHandler
import sys, io, json, cgi
import MolDisplay, molsql, molecule

# Create DB
db = molsql.Database(reset=True)
db.create_tables()

# Set our default element values
db['Elements'] = (1, 'H', 'Hydrogen', 'FFFFFF', '050505', '020202', 25)
db['Elements'] = (6, 'C', 'Carbon', '808080', '010101', '000000', 40)
db['Elements'] = (7, 'N', 'Nitrogen', '0000FF', '000005', '000002', 40)
db['Elements'] = (8, 'O', 'Oxygen', 'FF0000', '050000', '020000', 40)

# Add 3 molecules to the web page by default
fp = open('molecules/water-3D-structure.sdf')
db.add_molecule("Water", fp)

fp = open('molecules/caffeine-3D-structure-C.sdf')
db.add_molecule('Caffeine', fp)

fp = open('molecules/CID_31260.sdf')
db.add_molecule('Isopentanol', fp)

class myHandler(BaseHTTPRequestHandler):
    def do_GET(self):
        if self.path == "/":
            with open("index.html", "rb") as f: # main page
                html = f.read()
            self.send_response(200)
            self.send_header('Content-type', 'text/html')
            self.end_headers()
            self.wfile.write(html)

        elif self.path.endswith('.css'): # css
            with open("./public/css/styles.css", "rb") as f:
                css = f.read()
            # send CSS content
            self.send_response(200)
            self.send_header('Content-type', 'text/css')
            self.end_headers()
            self.wfile.write(css)

        elif self.path.endswith('.js'): # javascript
            with open("./public/js/scripts.js", "rb") as f:
                js = f.read()
            # send JS content
            self.send_response(200)
            self.send_header('Content-type', 'text/javascript')
            self.end_headers()
            self.wfile.write(js)

        elif self.path.endswith('.jpg'): # header pic
            with open("./public/images/header.jpg", 'rb') as f:
                content = f.read()
                self.send_response(200)
                self.send_header('Content-type', 'image/png')
                self.end_headers()
                self.wfile.write(content)

        # retrieves all molecules stored in the db
        elif self.path == "/retrieve-molecules":
            molecules_query = db.cursor.execute(f"""
                SELECT Molecules.NAME, COUNT(DISTINCT MoleculeAtom.ATOM_ID) AS numAtoms, COUNT(DISTINCT MoleculeBond.BOND_ID) AS numBonds
                FROM Molecules
                JOIN MoleculeAtom ON MoleculeAtom.MOLECULE_ID = Molecules.MOLECULE_ID
                JOIN Atoms ON MoleculeAtom.ATOM_ID = Atoms.ATOM_ID
                JOIN MoleculeBond ON MoleculeBond.MOLECULE_ID = Molecules.MOLECULE_ID
                JOIN Bonds ON MoleculeBond.BOND_ID = Bonds.BOND_ID
                GROUP BY Molecules.NAME;
            """).fetchall()

            molecules_list = []
            for molecule in molecules_query:
                molecule_values = {"name": molecule[0], "numAtoms": molecule[1], "numBonds": molecule[2]}
                molecules_list.append(molecule_values)

            self.send_response(200)
            self.send_header("Content-Type", "application/json")
            self.end_headers()
            self.wfile.write(json.dumps(molecules_list).encode())

        # retrieves all elements stored in the db
        elif self.path == "/retrieve-elements":
            elements_query = db.cursor.execute("""SELECT ELEMENT_NO, ELEMENT_CODE, ELEMENT_NAME, COLOUR1, COLOUR2, COLOUR3, RADIUS FROM ELEMENTS""").fetchall()
            elements_list = []

            for element in elements_query:
                element_values = {"Element No": element[0], "Element Code": element[1], "Element Name": element[2], "Colour 1": element[3], "Colour 2": element[4], "Colour 3": element[5], "Radius": element[6]}
                elements_list.append(element_values)

            self.send_response(200)
            self.send_header("Content-Type", "application/json")
            self.end_headers()
            self.wfile.write(json.dumps(elements_list).encode())
        else:
            self.send_error(404, "Error Found")
            self.end_headers()
            self.wfile.write(bytes("404: Error Not found", "utf-8"))

    def do_POST(self):
        # request to display a molecule
        if self.path == '/molecule':
            content_length = int(self.headers.get('Content-Length'))
            molecule_data = self.rfile.read(content_length)
            data = json.loads(molecule_data)
            molecule_name = data.get('molecule') # receive the molecule name

            # Build our default dictionaries
            MolDisplay.radius = db.radius()
            MolDisplay.element_name = db.element_name()
            MolDisplay.header += db.radial_gradients()

            print(MolDisplay.element_name)

            # load the molecule and generate svg
            mol = db.load_mol(molecule_name)
            mol.sort()
            svg = mol.svg() # create the svg

            print(svg)

            # send the svg back to the client
            self.send_response(200)
            self.send_header('Content-type', 'image/svg+xml')
            self.end_headers()
            self.wfile.write(bytes(svg, "utf-8"))

        # request to upload a new molecule
        elif self.path == "/upload-file":
            cgi.parse_header(self.headers['Content-Type'])
            # Parse the form data to get the file and molecule name
            form = cgi.FieldStorage(
                fp = self.rfile,
                headers = self.headers,
                environ = {'REQUEST_METHOD': 'POST'}
            )

            # receive the file object
            file_item = form['file']
            molecule_name = form.getvalue('moleculeName')

            # read the file contents
            file_contents = file_item.file.read()

            # convert to bytes in order to parse
            bytes_io = io.BytesIO(file_contents)
            data = io.TextIOWrapper(bytes_io)

            # add the molecule to the database
            db.add_molecule(molecule_name, data)
            self.send_response(200)
            self.end_headers()

        # request to add an element to the database
        elif self.path == "/add-element":
            # Read the request body as JSON data
            content_length = int(self.headers["Content-Length"])
            body = self.rfile.read(content_length)
            element_data = json.loads(body)

            # Insert the element into the Elements table
            db["Elements"] = (
                element_data["elementNum"],
                element_data["elementCode"],
                element_data["elementName"],
                element_data["colourOne"],
                element_data["colourTwo"],
                element_data["colourThree"],
                element_data["elementRadius"],
            )

            new_element = {
                "elementNum": element_data["elementNum"],
                "elementCode": element_data["elementCode"],
                "elementName": element_data["elementName"],
                "colourOne": element_data["colourOne"],
                "colourTwo": element_data["colourTwo"],
                "colourThree": element_data["colourThree"],
                "elementRadius": element_data["elementRadius"],
            }

            self.send_response(200)
            self.send_header("Content-type", "application/json; charset=utf-8")
            self.end_headers()
            self.wfile.write(json.dumps(new_element).encode())

        # request to remove an element from the database
        elif self.path == "/remove-element":
            content_length = int(self.headers["Content-Length"])
            body = self.rfile.read(content_length)
            element_data = json.loads(body)
            # print(element_data)
            element_code = element_data['elementCode']

            # deletes the row in the Elements table matching the element code
            db.cursor.execute(f"""
                DELETE FROM Elements WHERE ELEMENT_CODE = "{element_code}";
            """)

            self.send_response(200)
            self.send_header("Content-type", "application/json; charset=utf-8")
            self.end_headers()
            self.wfile.write(json.dumps({"success": True}).encode("utf-8"))

        elif self.path == "/rotate-molecule":
            content_length = int(self.headers["Content-Length"])
            body = self.rfile.read(content_length)
            rotate_data = json.loads(body)

            molecule_to_rotate = rotate_data['molecule'] # receive the molecule name

            try:
                x = int(rotate_data['x'])
            except:
                x = 0
            try:
                y = int(rotate_data['y'])
            except:
                y = 0
            try:
                z = int(rotate_data['z'])
            except:
                z = 0

            # receive the molecule to rotate
            mol = db.load_mol(molecule_to_rotate)

            # rotate the molecule
            mx = molecule.mx_wrapper(x, y, z)
            mol.xform(mx.xform_matrix)

            # receive the new SVG after rotation and send to client
            svg = mol.svg()

            self.send_response(200)
            self.send_header('Content-type', 'image/svg+xml')
            self.end_headers()
            self.wfile.write(bytes(svg, "utf-8"))
        else:
            self.send_error(404, "Error Found")
            self.end_headers()
            self.wfile.write(bytes("404: Error Not found", "utf-8"))

httpd = HTTPServer(('localhost', int(sys.argv[1])), myHandler)
httpd.serve_forever()


import io
import runoff_class as runoff
import unittest
from unittest import mock

class TestRunoff(unittest.TestCase):

    def setUp(self) -> None:
        self.candidates = ["Marta", "Joni", "Fran", "Linda"]
        self.voter_number = 5
        self.model = runoff.Runoff(self.candidates, self.voter_number)
        self.votes_to_cast = [
            ["Marta", "Joni", "Fran", "Linda"],
            ["Joni", "Marta", "Fran", "Linda"],
            ["Linda", "Marta", "Joni", "Fran"],
            ["Marta", "Joni", "Fran", "Linda"],
            ["Marta", "Fran", "Linda", "Joni"],
        ]

        # cast votes
        for voter_index in range(0, self.voter_number):
            for name in self.votes_to_cast[voter_index]:
                self.model.vote(voter_index, name)

    def test_that_candidates_are_populated_correctly(self):
        candidate_names = self.model.candidates.keys()
        for name, person in zip(self.candidates, candidate_names):
            self.assertEqual(name, person)

    def test_that_invalid_candidates_are_caught(self):
        """Checks that the candidate being voted for is in the list of
        valid candidates"""

        people_validity = {
            "Marta": True,
            "Benoit": False,
            "Joni": True,
            "Ransom": False,
        }

        model = self.model

        for person, validity in people_validity.items():
            self.assertEqual(validity, model.candidate_is_valid(person))

    def test_that_voter_preferences_are_recorded_correctly(self):
        """Checks that ranks of candidates chosen by each voter are
        accurately recorded."""

        expected = self.votes_to_cast
        self.assertEqual(expected, self.model.voter_prefs)

    def test_that_votes_are_tabulated_correctly(self):
        """Checks that the vote counts for the highest preference candidates
        are updated accurately."""

        self.model.tabulate()

        expected = {"Marta": 3, "Joni": 1, "Fran": 0, "Linda": 1}
        actual = self.model.candidates.values()
        for vote, candidate_vote in zip(expected.values(), actual):
            self.assertEqual(vote, candidate_vote.votes)

    def test_that_the_smallest_vote_is_reported_correctly(self):
        """Checks that the smallest number of votes that any remaining (
        non-eliminated) candidates is returned."""

        self.model.tabulate()

        # Fran has no votes.
        expected_minimum = 0

        self.assertEqual(expected_minimum, self.model.find_minimum())

        votes_to_cast = [
            ["Marta", "Joni", "Fran", "Linda"],
            ["Joni", "Marta", "Fran", "Linda"],
            ["Linda", "Marta", "Joni", "Fran"],
            ["Fran", "Joni", "Marta", "Linda"],
            ["Marta", "Fran", "Linda", "Joni"],
        ]

        # reset voter prefs
        self.model.voter_prefs = [[] for x in range(0, self.voter_number)]

        # cast votes
        for voter_index in range(0, self.voter_number):
            for name in votes_to_cast[voter_index]:
                self.model.vote(voter_index, name)

        self.model.tabulate()

        expected_minimum = 1

        self.assertEqual(expected_minimum, self.model.find_minimum())

    def test_that_a_tied_election_is_recognized(self):
        voter_number = 3
        self.model = runoff.Runoff(self.candidates, voter_number)

        votes_to_cast = [
            ["Joni", "Marta", "Fran", "Linda"],
            ["Fran", "Marta", "Fran", "Linda"],
            ["Joni", "Linda", "Joni", "Fran"],
        ]

        # Artificially eliminate Joni
        self.model.candidates["Joni"].eliminated = True

        # cast votes
        for voter_index in range(0, voter_number):
            for name in votes_to_cast[voter_index]:
                self.model.vote(voter_index, name)
        minimum_vote = self.model.find_minimum()

        # Marta has 0 votes
        self.assertEqual(minimum_vote, 0)

        self.assertTrue(self.model.is_tie(minimum_vote))

        # reset

        new_candidates = ["Joni", "Marta", "Meg", "Fran", "Linda", "Ransom"]
        new_voter_number = 4
        self.model = runoff.Runoff(new_candidates, new_voter_number)

        votes_to_cast = [
            ["Joni", "Marta", "Meg", "Fran", "Linda", "Ransom"],
            ["Fran", "Joni", "Ransom", "Marta", "Meg", "Linda"],
            ["Joni", "Linda", "Marta", "Ransom", "Fran", "Meg"],
            ["Meg", "Linda", "Joni", "Marta", "Ransom", "Fran"],
        ]

        # cast votes
        for voter_index in range(0, new_voter_number):
            for name in votes_to_cast[voter_index]:
                self.model.vote(voter_index, name)

        # Artificially eliminate Joni and Fran
        self.model.candidates["Joni"].eliminated = True
        self.model.candidates["Fran"].eliminated = True

        self.model.tabulate()

        minimum_vote = self.model.find_minimum()

        # Joni and Fran's 0 votes do not count, because they are eliminated.
        self.assertEqual(minimum_vote, 1)

        self.assertTrue(self.model.is_tie(minimum_vote))

    def test_that_a_candidate_in_last_place_is_eliminated(self):
        """Checks that a candidate/candidates with the fewest number of votes
        are eliminated."""

        # reset

        new_candidates = ["Marta", "Joni", "Fran", "Linda", "Meg"]
        new_voter_number = 4
        self.model = runoff.Runoff(new_candidates, new_voter_number)

        votes_to_cast = [
            ["Marta", "Joni", "Fran", "Linda", "Meg"],
            ["Joni", "Marta", "Meg", "Fran", "Linda"],
            ["Linda", "Marta", "Joni", "Meg", "Fran"],
            ["Fran", "Joni", "Meg", "Marta", "Linda"],
        ]

        # cast votes
        for voter_index in range(0, new_voter_number):
            for name in votes_to_cast[voter_index]:
                self.model.vote(voter_index, name)

        self.model.tabulate()

        self.model.eliminate(self.model.find_minimum())

        eliminated = {
            "Marta": False,
            "Joni": False,
            "Fran": False,
            "Linda": False,
            "Meg": True,
        }

        for expected, actual in zip(eliminated.values(),
                                    self.model.candidates.values()):
            self.assertEqual(expected, actual.eliminated)

        self.model.tabulate()

        self.model.eliminate(self.model.find_minimum())

        for expected, actual in zip(eliminated.values(),
                                    self.model.candidates.values()):
            self.assertEqual(expected, actual.eliminated)

    def test_that_winners_are_printed_correctly(self):

        self.model.tabulate()

        with mock.patch("sys.stdout", new=io.StringIO()) as out:
            self.model.print_winner()

        assert out.getvalue() == "Marta\n"

        # reset

        candidates = ["Alice", "Bob", "Charlie"]
        new_voter_number = 9
        self.model = runoff.Runoff(candidates, new_voter_number)

        votes_to_cast = [
            ["Alice", "Bob", "Charlie"],
            ["Alice", "Bob", "Charlie"],
            ["Bob", "Alice", "Charlie"],
            ["Bob", "Alice", "Charlie"],
            ["Bob", "Alice", "Charlie"],
            ["Charlie", "Alice", "Bob"],
            ["Charlie", "Alice", "Bob"],
            ["Charlie", "Bob", "Alice"],
            ["Charlie", "Bob", "Alice"],
        ]

        # cast votes
        for voter_index in range(0, new_voter_number):
            for name in votes_to_cast[voter_index]:
                self.model.vote(voter_index, name)

        self.model.tabulate()

        eliminated = {
            "Alice": True,
            "Bob": False,
            "Charlie": False,
        }

        self.model.eliminate(self.model.find_minimum())

        for expected, actual in zip(eliminated.values(),
                                    self.model.candidates.values()):
            self.assertEqual(expected, actual.eliminated)

        self.model.tabulate()

        with mock.patch("sys.stdout", new=io.StringIO()) as out:
            self.model.print_winner()

        assert out.getvalue() == "Bob\n"