from flask import Flask
from flask_cors import CORS
from flask import request, jsonify, abort, Response
import json
import csv
from urllib.parse import urlparse
import re
import sys
from joblib import dump, load
import spacy
import mechanicalsoup
from textstat import textstat

app = Flask(__name__)
CORS(app)
# app.config["CORS_SUPPORTS_CREDENTIALS"] = True

@app.route("/")
def home():
    return Response(headers={'Access-Control-Allow-Origin':'*'})

@app.route('/endpoint/', methods=['GET','POST'])
def create_task():
    if request.method == "POST":
        content = request.get_json()
        news = filter_for_news(content)
        return get_biases_for_news(news)
    else:
        return jsonify({'success':True})

@app.route('/endpoint2/', methods=['GET','POST'])
def log_sites():
    if request.method == "POST":
        content = request.get_json()
        save_html(content)
        # print("ENDPOINT 2 HERE")
        return jsonify({'success':True})
        # news = filter_for_news(content)
        # return get_biases_for_news(news)
    else:
        return jsonify({'success':False})

def save_html(html_json):
    start = html_json.index('"body": "') + len('"body": "')
    url = html_json[20:(start-12)]
    if is_it_news(url):
        html = html_json[start:-4]
        with open('saved_html.csv', 'r+') as csvfile:
            csv.field_size_limit(sys.maxsize)
            csv_reader = csv.reader(csvfile, delimiter=",")
            already_saved = 0
            for row in csv_reader:
                # print(row[0])
                if row[0] == url:
                    already_saved = 1
                    break
            if not already_saved:
                clf = load('../bias_classifier/filename.joblib')
                bias = clf.predict([get_classifiers(html)])
                # do I want to convert the classifier here?
                writer = csv.writer(csvfile, delimiter=',', quotechar='"', quoting=csv.QUOTE_ALL)
                writer.writerow([url, bias, html])

# {'title': 'Technology - The New York Times', 'url': 'https://www.nytimes.com/section/technology',
# 'lastVisit': 'Mon Mar 01 2021 20:27:36 GMT-0500 (Eastern Standard Time)', 'count': '1'}
def get_biases_for_news(news_dict):
    with open('../all_bias.csv', 'r', newline='') as csvfile:
        spamreader = csv.reader(csvfile, delimiter=',')
        biasdict = {rows[0]:[rows[1], rows[2], rows[3], rows[4]] for rows in spamreader}
    data = {}
    data['history'] = []
    with open('saved_html.csv', 'r', newline='') as csvfile:
        csv.field_size_limit(sys.maxsize)
        spamreader = csv.reader(csvfile, delimiter=',')
        classifier_dict = {rows[0]:rows[1] for rows in spamreader}
    for entry in news_dict:
        parsed = urlparse(entry['url'])
        url = parsed.netloc
        if re.match(r"^www\..+\.", url):
                url = re.sub(r"^www\.", "", url, 1)
        classifier_bias = "none"
        if entry['url'] in classifier_dict:
            classifier_bias = classifier_dict[entry['url']]
        if url in biasdict:
            data['history'].append({
                'domain': url,
                'mbfc_bias' : biasdict[url][1].replace(" ", ""),
                'allsides_bias': biasdict[url][2].replace(" ", ""),
                'classifier_bias': classifier_bias,
                'mbfc_fake': biasdict[url][3].replace(" ", ""),
                'source_title': biasdict[url][0],
                'title': entry['title'],
                'url': entry['url']
            })
        else:
            data['history'].append({
                'domain': url,
                'mbfc_bias' : "none",
                'allsides_bias': "none",
                'classifier_bias': classifier_bias,
                'mbfc_fake': "none",
                'source_title': url,
                'title': entry['title'],
                'url': entry['url']
            })
    my_json = json.dumps(data)
    return my_json

def filter_for_news(history_dict):
    with open('../all_news.csv', 'r', newline='') as csvfile:
        spamreader = csv.reader(csvfile, delimiter=',')
        mydict = {rows[0]:"news" for rows in spamreader}
    news = []
    for entry in history_dict['history']:
        parsed = urlparse(entry['url'])
        url = parsed.netloc
        if re.match(r"^www\..+\.", url):
                url = re.sub(r"^www\.", "", url, 1)
        if url in mydict:
            news.append(entry)
    return news

def is_it_news(url):
    with open('../all_news.csv', 'r', newline='') as csvfile:
        spamreader = csv.reader(csvfile, delimiter=',')
        mydict = {rows[0]:"news" for rows in spamreader}
        parsed = urlparse(url)
        url = parsed.netloc
        if re.match(r"^www\..+\.", url):
                url = re.sub(r"^www\.", "", url, 1)
        if url in mydict:
            return True
        else:
            return False

def get_classifiers(text, url=False):
    nlp = spacy.load("en_core_web_sm")
    if url:
        text = get_text(text)
    # tokens = get_phrases()
    # tokens = ['19', '2020', 'ad', 'ago', 'american', 'april 2021', 'best', 'biden', 'black', 'business', 'chauvin', 'city', 'com', 'content', 'covid', 'covid 19', 'culture', 'day', 'don', 'facebook', 'family', 'floyd', 'force', 'government', 'health', 'law', 'like', 'make', 'military', 'mr', 'national', 'party', 'pm', 'police', 'politics', 'president', 'prince', 'reply', 'say', 'says', 'share', 'social', 'subscribe', 'support', 'trump', 'twitter', 'vaccine', 'video', 'white', 'york']
    # tokens = ['according', 'american', 'campaign', 'clinton', 'country', 'day', 'did', 'don', 'donald', 'donald trump', 'going', 'government', 'house', 'just', 'know', 'like', 'make', 'mr', 'mr trump', 'national', 'new', 'news', 'obama', 'party', 'people', 'police', 'political', 'president', 'republican', 'say', 'says', 'state', 'states', 'think', 'time', 'told', 'trump', 'twitter', 'united', 'united states', 'want', 'way', 'white', 'women', 'world', 'year', 'years']
    tokens = ['000', 'according', 'american', 'campaign', 'city', 'clinton', 'country', 'day', 'did', 'don', 'donald', 'donald trump', 'going', 'government', 'house', 'including', 'just', 'know', 'like', 'make', 'million', 'mr', 'national', 'new', 'new york', 'news', 'obama', 'officials', 'people', 'percent', 'police', 'president', 'say', 'says', 'state', 'states', 'think', 'time', 'told', 'trump', 'united', 'united states', 'way', 'week', 'white', 'world', 'year', 'years']
    result = []

    lex_count = textstat.lexicon_count(text)
    sent_count = textstat.sentence_count(text)
    flesch_score = textstat.flesch_reading_ease(text)
    smog_score = textstat.smog_index(text)
    auto_readability_score = textstat.automated_readability_index(text)

    for word in tokens:
        if word in text:
            result.append(1)
        else:
            result.append(0)

    doc = nlp(text)
    adj = adp = adv = aux = conj = det = intj = noun = num = part = pron = propn = punct = sconj = sym = verb = x = 0
    div = len(doc)
    for token in doc:
        if token.pos_ == "ADJ":
            adj+=1
        elif token.pos_ == "APD":
            adp +=1
        elif token.pos_ == "ADV":
            adv+=1
        elif token.pos_ == "AUX":
            aux+= 1
        elif token.pos_ == "NOUN":
            noun+= 1
        elif token.pos_ == "PRON":
            pron+=1
        elif token.pos_ == "VERB":
            verb+=1
        elif token.pos == "CONJ":
            conj+=1
        elif token.pos == "DET":
            det+=1
        elif token.pos == "INTJ":
            intj+=1
        elif token.pos == "NUM":
            num+=1
        elif token.pos == "PART":
            part+=1
        elif token.pos == "PROPN":
            propn+=1
        elif token.pos == "PUNCT":
            punct+=1
        elif token.pos == "SCONJ":
            sconj+=1
        elif token.pos == "SYM":
            sym+=1
        elif token.pos == "X":
            x+=1

    add = [lex_count, sent_count, flesch_score, smog_score, auto_readability_score, adj/div, adp/div, adv/div, aux/div, noun/div, pron/div, verb/div, conj/div, det/div, intj/div, num/div, part/div, propn/div, punct/div, sconj/div, sym/div, x/div]
    # col_names = ['lex_count', 'sent_count', 'flesch_score', 'smog_score', 'auto_readability_score', 'adj/div', 'adp/div', 'adv/div', 'aux/div', 'noun/div', 'pron/div', 'verb/div', 'conj/div', 'det/div', 'intj/div', 'num/div', 'part/div', 'propn/div', 'punct/div', 'sconj/div', 'sym/div', 'x/div']

    result.extend(add)
    return result

def get_text(url):
    browser = mechanicalsoup.Browser()
    page = browser.get(url)
    body = page.soup.body
    total_strings = " "
    total_strings = total_strings.join(body.strings)
    if "404" in total_strings:
        raise Exception("404 error")
    return total_strings

if __name__ == '__main__':
    app.run(host='localhost', debug=True)


import typing

class UnionFind():
    def __init__(self, n: int) -> typing.NoReturn:
        self.__data = [-1] * n

    def find(self, u: int) -> int:
        d = self.__data
        if d[u] < 0: return u
        d[u] = self.find(d[u])
        return d[u]

    def size(self, u: int) -> int:
        return -self.__data[self.find(u)]

    def unite(self, u: int, v: int) -> typing.NoReturn:
        u, v = self.find(u), self.find(v)
        if u == v: return
        d = self.__data
        if d[u] > d[v]: u, v = v, u
        d[u] += d[v]
        d[v] = u

def main() -> typing.NoReturn:
    n, m = map(int, input().split())
    uf = UnionFind(n)
    for _ in range(m):
        a, b = map(int, input().split())
        a -= 1
        b -= 1
        uf.unite(a, b)

    print(max(uf.size(i) for i in range(n)))

main()


import os
import time
import boto3
import rasterio
import threading
import pandas as pd

from pathlib import Path
from pycksum import cksum
from botocore import UNSIGNED
from collections import deque, Counter
from botocore.config import Config

class DownloadCallback(object):

    def __init__(self, filename, size):
        self._filename = filename
        self._remain = size
        self._lock = threading.Lock()

    def __call__(self, bytes_amount):
        with self._lock:
            self._remain -= bytes_amount

class ChallengeCache:

    def __init__(self, directory, limit=100):
        self._s3 = boto3.client('s3', config=Config(signature_version=UNSIGNED))
        self._directory = Path(directory)
        self._limit = limit
        self._files = deque()
        self._lock = threading.Lock()
        self.refresh()

    def refresh(self):
        with self._lock:
            for path in self._directory.glob('**/*'):
                if path.is_file():
                    self._files.append(str(path))

    def append(self, path):
        with self._lock:
            self._files.append(path)

    def exists(self, path):
        with self._lock:
            return path in self._files

    def get_file(self, s3path, chksum=None, size=0, refresh=False, ntries=3):
        tmp = s3path.split('/')
        if tmp[0] != 's3:':
            return s3path

        object_path = '/'.join(tmp[3:])
        local_path = self._directory / object_path

        if local_path.exists():
            if not refresh:
                return str(local_path)
            local_path.unlink()
            time.sleep(0.1)

        local_path.parent.mkdir(exist_ok=True)

        for _ in range(ntries):
            cb = DownloadCallback(str(local_path), size) if size > 0 else None
            with local_path.open('wb') as (fobj):
                bucket_name = tmp[2]
                self._s3.download_fileobj(bucket_name,
                  object_path,
                  fobj,
                  Callback=cb)

            while not local_path.exists():
                time.sleep(0.1)

            if chksum is None or chksum == cksum(open(local_path, 'rb')):
                self.append(str(local_path))
                return str(local_path)

        raise ValueError(f"Checksum Error: {s3path}")

    def expire(self):
        with self._lock:
            n = len(self._files)
            while len(self._files) > self._limit:
                f = self._files.popleft()
                if os.path.exists(f):
                    os.remove(f)

            return n - len(self._files)

    def cleanup(self):
        with self._lock:
            while len(self._files) > 0:
                f = self._files.popleft()
                if os.path.exists(f):
                    os.remove(f)

class ChallengeData(ChallengeCache):

    def __init__(self, config):
        super().__init__(config.base_dir + '/raw', 800)
        self._base = Path(config.base_dir)
        self._s3path = f"s3path_{config.s3region}"
        self._s3prefix = f"s3://{config.s3bucket}-{config.s3region}"
        self._id_str = config.id_str

        self.nodata = config.nodata
        self.df_meta = self._read_meta(self._s3prefix, config.metadata)

        df_fold_path = self._base / 'image_folds.csv'
        if df_fold_path.exists():
            self.df_fold = pd.read_csv(df_fold_path, index_col=0)
        else:
            self.df_fold = self._make_folds(config.nfolds)
            self.df_fold.to_csv(df_fold_path)

    def subdir(self, name):
        child = self._base/name
        child.mkdir(exist_ok=True)
        return child

    def resolve(self, path):
        path = path if isinstance(path, Path) else Path(path)
        if len(path.parts) > 1:
            return path.resolve()
        else:
            return (self._base / path).resolve()

    def _read_meta(self, s3prefix, metadata):
        df = {}
        for k in ('features', 'train_agbm'):
            meta = self.get_file(f"{s3prefix}/{metadata[k]}")
            df[k] = pd.read_csv(meta)

        if metadata.features_stats:
            latest_stats = self.resolve(metadata.features_stats)
            if latest_stats.exists():
                stats = pd.read_csv(latest_stats).drop((self._s3path), axis=1)
                stats = stats.loc[(stats.defects > -1)]
                feats = df['features'].loc[stats.index]
                df['features'] = feats.join((stats.set_index('filename')),
                  on='filename')
        return df

    def _make_folds(self, nfolds):
        df_feat = self.df_meta['features']
        df_agbm = self.df_meta['train_agbm']

        if 'defects' in df_feat.columns:
            df_images = df_feat.loc[((df_feat.defects == 0) | (df_feat.split == 'test'))]
        else:
            df_images = df_feat

        chip_ids = df_images.groupby(['chip_id', 'satellite']).count() > 0
        chip_ids = Counter(id for id, _ in chip_ids.index)
        chip_ids = [id for id, cnt in chip_ids.items() if cnt == 2]

        df_feat = df_feat.loc[df_feat.chip_id.isin(chip_ids)]

        train_ids = df_feat.loc[(df_feat.split == 'train')][self._id_str].unique()
        train_ids = list(set.intersection(set(train_ids), set(df_agbm[self._id_str].unique())))
        print(len(train_ids), 'training chips')

        test_ids = df_feat.loc[(df_feat.split == 'test')][self._id_str].unique()
        print(len(test_ids), 'testing chips')

        df = pd.DataFrame({'fold': -1}, index=train_ids)
        df = df.sample(frac=1.0, random_state=42)

        size = (df.shape[0] + nfolds - 1) // nfolds
        for fold in range(nfolds):
            df.iloc[fold * size:(fold + 1) * size, 0] = fold

        return pd.concat([df, pd.DataFrame({'fold': -1}, index=test_ids)])

    def get_means(self, chip_ids=None):
        if chip_ids is None:
            chip_ids = self.df_fold.index.values

        df = self.df_meta['features']
        df = df.loc[df.chip_id.isin(chip_ids)]

        means = {}
        for sentinel in ('S1', 'S2'):
            cols = [f"mean_b{i}" for i in range(len(self._nodata[sentinel.lower()]))]
            vals = df.loc[(df.satellite == sentinel, cols)].mean().values
            means[sentinel.lower()] = vals

        return means

    def load_image(self, s3path, chksum=None, size=0, refresh=False):
        tmp = s3path.split('/')
        if tmp[0] == 's3':
            filename = tmp[(-1)]
            if filename.lower().endswith('_agbm.tif'):
                df = self.df_meta['train_agbm']
            else:
                df = self.df_meta['features']

            if size < 1:
                values = df.loc[(df.filename == filename, 'size')].values
                if len(values) == 1:
                    size = values[0]
                else:
                    raise ValueError(f"Unknown file size: {filename}")

            if chksum is None:
                values = df.loc[(df.filename == filename, 'cksum')].values
                if len(values) == 1:
                    chksum = values[0]

        raw_path = None
        while raw_path is None or not os.path.exists(raw_path):
            raw_path = self.get_file(s3path, chksum, size, refresh)

        with rasterio.open(raw_path) as (file):
            return (file.read(), raw_path)

    def get_stats(self, names=None, chip_ids=None):
        names = ['mean', 'std'] if names is None else names
        names = [names] if isinstance(names, str) else names

        if chip_ids is None:
            chip_ids = self.df_fold.index.values

        df = self.df_meta['features']
        df = df.loc[df.chip_id.isin(chip_ids)]

        stats = {}
        for name in names:
            stat = {}
            for sentinel in ('s1', 's2'):
                cols = [f"{name}_b{i}" for i in range(len(self.nodata[sentinel]))]
                rows = df.satellite == sentinel.upper()
                if df.loc[(rows, cols)].size > 0:
                    vals = df.loc[(rows, cols)].mean().values
                    stat[sentinel] = list(float(v) for v in vals)

            stats[name] = stat

        return stats

    def get_ids(self, fold):
        return self.df_fold.loc[(self.df_fold.fold == fold)].index

    def get_item(self, id):
        df_feat = self.df_meta['features']
        df_agbm = self.df_meta['train_agbm']

        s3paths_feat = df_feat.loc[(df_feat[self._id_str] == id)][self._s3path].values
        if self.df_fold.at[(id, 'fold')] != -1:
            s3paths_agbm = df_agbm.loc[(df_agbm[self._id_str] == id)][self._s3path].values
        else:
            s3paths_agbm = None

        return (s3paths_feat, s3paths_agbm[0] if s3paths_agbm else None)


from sklearn.datasets import fetch_openml
from PIL import Image
import matplotlib
import matplotlib.pyplot as plt
import cv2
import numpy as np
import math
import pytesseract
import pyautogui
import matplotlib.image as mpimg

mnist = fetch_openml('mnist_784')
data = mnist['data']
target = mnist['target']

import numpy as np
data_train = data[:60000]
target_train = target[:60000]
data_test = data[60000:]
target_test_temp = target[60000:]

shuffle = np.random.permutation(60000)
data_train = data_train[shuffle]
target_train_temp = target_train[shuffle]

target_train = []
for i in range(len(target_train_temp)):
    temp = []
    for j in range(10):
        if j == int(target_train_temp[i]):
            temp.append(1)
        else:
            temp.append(0)
    target_train.append(temp)
target_train = np.array(target_train)

target_test = []
for i in range(len(target_test_temp)):
    temp = []
    for j in range(10):
        if j == int(target_test_temp[i]):
            temp.append(1)
        else:
            temp.append(0)
    target_test.append(temp)
target_test = np.array(target_test)

#DNN
from keras import Sequential
from keras.layers import Dense, Activation
model = Sequential()
#加一層fully connected layer(dense)
#model.add(Dense(input_dim = 28*28, units = 500))
model.add(Dense(500, input_dim = 28*28))
model.add(Activation('sigmoid'))
#再加一層fully connected layer(dense), 不用input, 跟前一層output一樣
model.add(Dense(500))
model.add(Activation('sigmoid'))
model.add(Dense(10))
model.add(Activation('softmax'))

#evaluate model好壞
#configuration(optimizer)
model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])

#batch越大，速度越快，但是結果會爛掉
#batch越小，速度越慢，gpu平行運算加速效果不明顯
model.fit(data_train, target_train, batch_size=100, epochs=20)

score = model.evaluate(data_test, target_test)

im = Image.fromarray(data_test[4].reshape(28,28))
plt.imshow(im)

predict = model.predict(data_test)[4]
np.where(predict == np.amax(predict))

# read handwrite image
data = Image.open(r'C:\Users\yt335\Desktop\handwriting number.png').convert('L')
data_np = np.array(data).reshape(1,784)
data_np = data_np/255
predict = model.predict(data_np)
np.where(predict == np.amax(predict))

#OCR零件文字辨識
imagepath = 'test4.jpg'

img = cv2.imread(imagepath)
img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
img = cv2.cvtColor(img, cv2.COLOR_RGB2GRAY)
img = cv2.resize(img, (28, 28))
#blueImg = img[:,:,0]
#b = get_blue(img)
#img = cv2.cvtColor(b, cv2.IMREAD_GRAYSCALE)
ret,img = cv2.threshold(img, 130,255,cv2.THRESH_BINARY_INV)

dst = cv2.fastNlMeansDenoising(img,None,10,7,21)

median = cv2.medianBlur(dst, 5)
kernel_size = 3
img = cv2.GaussianBlur(median,(kernel_size, kernel_size), 0)
#cv2.imwrite('output.jpg', img)
img = img/255

plt.imshow(img)

test = np.array(img).reshape(1,784)
predict = model.predict(test)
np.where(predict == np.amax(predict))


from anytree import Node, RenderTree

def main():
    lines = open("day7/inputs.txt").readlines()
    n = Node(name="Root", type="dir")
    root = n
    current_node: Node = n
    for line in lines:
        line = line.rstrip()
        if line.startswith("$"):
            if line.startswith("$ cd"):
                dir = line.split(" ")[2]
                if line == "$ cd /":
                    pass
                elif dir == "..":
                    current_node = current_node.parent
                else:
                    n = next(x for x in current_node.children if x.name == dir)
                    current_node = n
            elif line.startswith("$ ls"):
                pass
        else:
            if line.startswith("dir"):
                dir_name = line.split(" ")[1]
                n = Node(name=dir_name, parent=current_node, type="dir")
            else:
                size, file_name = line.split(" ")
                n = Node(name=file_name, parent=current_node, type="file", size=size)

    result = RenderTree(root)
    total_size = 0
    for node in root.descendants:
        if node.type == "dir":
            size = 0
            for n in node.descendants:
                if n.type == "file":
                    size += int(n.size)
            if size <= 100000:
                total_size += size
    print(total_size)

if __name__ == "__main__":
    main()


import dtlpy as dl
import time
import jwt
import random
import logging
import os
from dtlpy import examples

try:
    # for local import
    from tests.env_from_git_branch import get_env_from_git_branch
except ImportError:
    # for remote import
    from env_from_git_branch import get_env_from_git_branch

try:
    # for local import
    from tests.env_from_git_branch import get_env_from_git_branch
except ImportError:
    # for remote import
    from env_from_git_branch import get_env_from_git_branch

logging.basicConfig(level='DEBUG')
_, base_env = get_env_from_git_branch()
dl.setenv(base_env)
# check token
payload = jwt.decode(dl.token(), algorithms=['HS256'], verify=False,
                     options={'verify_signature': False})
if payload['email'] not in ['oa-test-4@dataloop.ai', 'oa-test-1@dataloop.ai', 'oa-test-2@dataloop.ai',
                            'oa-test-3@dataloop.ai']:
    assert False, 'Cannot run test on user: "{}". only test users'.format(payload['email'])

TEST_DIR = os.path.dirname(os.path.realpath(__file__))
ASSETS_PATH = os.path.join(TEST_DIR, 'assets')
image_path = os.path.join(ASSETS_PATH, '0000000162.jpg')
annotations_path = os.path.join(ASSETS_PATH, 'annotations_new.json')

project = dl.projects.create('to-delete-test-project_examples_tester_{}'.format(random.randrange(1000, 100000)))
dataset = project.datasets.create('dataset_examples_tester_{}'.format(random.randrange(1000, 100000)))
item = dataset.items.upload(local_path=image_path, local_annotations_path=annotations_path)

# add labels
examples.add_labels.main(project_name=project.name, dataset_name=dataset.name)

# add metadata to item
examples.add_metadata_to_item.main(project_name=project.name, dataset_name=dataset.name, item_path=image_path)

# annotation convert to voc
second_project = dl.projects.create('to-delete-test-project_examples_tester_{}'.format(random.randrange(1000, 100000)))
second_dataset = second_project.datasets.create('dataset_examples_tester_{}'.format(random.randrange(1000, 100000)))
time.sleep(1)
second_item = second_dataset.items.upload(local_path=image_path)
examples.copy_annotations.main(first_project_name=project.name,
                               second_project_name=second_project.name,
                               first_dataset_name=dataset.name,
                               second_dataset_name=second_dataset.name,
                               first_remote_filepath=item.filename,
                               second_remote_filepath=second_item.filename)

# copy folder
examples.copy_folder.main(first_project_name=project.name,
                          second_project_name=second_project.name,
                          first_dataset_name=dataset.name,
                          second_dataset_name=second_dataset.name)

# show item and mask
examples.show_item_and_mask.main(project_name=project.name,
                                 dataset_name=dataset.name,
                                 item_remote_path=item.filename)

# show item and mask
examples.upload_items_with_modalities.main(project_name=project.name,
                                           dataset_name=dataset.name)
project.delete(True, True)
second_project.delete(True, True)


import numpy as np
import cv2
import cv2.aruco as ar
import math
import serial
import time

time.sleep(2)
cv2.namedWindow('Frame')

x_center=0
y_center=0
current_coord=[0,0]
angle=0
anglenew=0
anglep=0
anglet=0
def click_event(event, x, y, flags, params):
    global x_center, y_center,current_coord
    # checking for left mouse clicks
    if event == cv2.EVENT_LBUTTONDOWN:

        current_coord[0] = x
        current_coord[1] = y
        # displaying the coordinates
        # on the Shell
        #print('x =', x, ' ', 'y =', y)
        co = [x, y]
    cv2.imshow('Frame', frame)

def angle_diff():
    global anglenew, angle,anglep,anglet
    diff = angle - anglenew
    if abs(diff)<180:
        diff=diff
    else:
        if diff>0:
            diff=diff-360
        else:
            diff=diff+360
    return diff

arduino = serial.Serial(port='COM7', baudrate=115200, timeout=.1)
def write_read(x):
    arduino.write(bytes(x, 'utf-8'))
    time.sleep(0.05)

def doprocess(img):
    global x_center, y_center,angle,anglep,anglet
    gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
    aruco_dict = ar.Dictionary_get(ar.DICT_6X6_250)
    parameters = ar.DetectorParameters_create()  #question?
    corners, ids, _ = ar.detectMarkers(gray, aruco_dict, parameters=parameters)
    if ids is not None :
        dict = {}

        for i in range(0, len(ids)):
            dict[ids[i][0]] = corners[i][0]

        for id in dict:
            grey_x = int(dict[id][0][0]);grey_y = int(dict[id][0][1])
            green_x =int (dict[id][1][0]);green_y =int (dict[id][1][1])  ###question
            pink_x =int (dict[id][2][0]);pink_y =int (dict[id][2][1])
            white_x =int(dict[id][3][0]);white_y =int (dict[id][3][1])

            cv2.circle(img,(green_x,green_y),6,(0,256,0),-1)
            cv2.circle(img,(grey_x,grey_y),6,(125,125,125),-1)  ###question
            cv2.circle(img,(pink_x,pink_y),6,(180,105,255),-1)
            cv2.circle(img,(white_x,white_y),6,(255,255,255),-1)
            x_center = int(((grey_x+pink_x)/2+(white_x+green_x)/2)/2)
            y_center = int(((grey_y+pink_y)/2+(white_y+green_y)/2)/2)

            cv2.circle(img,(int(x_center),int(y_center)),6,(0,0,255),-1)
            x_mid = int((grey_x+green_x)/2)
            y_mid = int((grey_y+green_y)/2)
            cv2.line(img,(int(x_center),int(y_center)),(int(x_mid),int(y_mid)),(255,0,0),3)
            cv2.putText(img,f"{id}",(int(pink_x+10),int(pink_y)+10),cv2.FONT_HERSHEY_SIMPLEX,0.9,(0,0,255),2)
            anglep = round(math.degrees(math.atan2((white_y-grey_y),(grey_x-white_x))))
            if angle>=0:
                angle=anglep
            else:
                angle=anglep+360
            cv2.putText(frame, 'Aruco_angle', (50, 80), cv2.FONT_HERSHEY_SIMPLEX, 0.8, (20,50,250), 2)
            cv2.putText(img,f"{angle}",(50, 100),cv2.FONT_HERSHEY_SIMPLEX,0.8,(20,50,250),2)

            anglet = round(math.degrees(math.atan2((y_center - current_coord[1]), (current_coord[0] - x_center))))
            distance = round(math.sqrt((y_center - current_coord[1])**2 + (current_coord[0] - x_center)**2))
            cv2.putText(frame, 'Distance', (50, 200), cv2.FONT_HERSHEY_SIMPLEX, 0.8, (200, 0, 200), 2)
            cv2.putText(frame, str(distance), (50, 220), cv2.FONT_HERSHEY_SIMPLEX, 0.8, (200, 0, 200), 2)
            #print(current_coord[0], current_coord[1])
            if anglet >= 0:
                anglenew = anglet
            else:
                anglenew = anglet + 360
            diff = angle - anglenew
            if abs(diff) < 180:
                diff = diff
            else:
                if diff > 0:
                    diff = diff - 360
                else:
                    diff = diff + 360
            finalangle = diff
            command = ""
            if(diff>10 and distance>50):
                command = "right"
            elif (diff<-10 and distance>50):
                command = "left"
            elif (abs(diff)<=10 and distance>50):
                command = "forward"
            else:
                command = "stop"
            write_read(command)
            cv2.putText(frame, 'Angle_diff', (50, 130), cv2.FONT_HERSHEY_SIMPLEX, 0.8, (200, 0, 200), 2)
            cv2.putText(frame, str(finalangle), (50, 150), cv2.FONT_HERSHEY_SIMPLEX, 0.8, (200, 0, 200), 2)
            cv2.putText(frame, 'Click_angle', (50, 20), cv2.FONT_HERSHEY_SIMPLEX, 0.8, (0, 0, 0), 2)
            cv2.putText(frame, str(anglenew), (50, 50), cv2.FONT_HERSHEY_SIMPLEX, 0.8, (0, 0, 0), 2)
            cv2.line(frame, (int(x_center), int(y_center)), (int(current_coord[0]), int(current_coord[1])), (255, 0, 0),
                     3)
            cv2.putText(frame, '-ve for Anti_Clk and +ve for Clk', (200, 20), cv2.FONT_HERSHEY_SIMPLEX, 0.8,
                        (0, 0, 200), 2)
    return img

cap = cv2.VideoCapture(1)
#result = cv2.VideoWriter('RESULT_VIDEO.avi',cv2.VideoWriter_fourcc(*'MJPG'),20,(int(cap.get(3)),int(cap.get(4))))
cv2.setMouseCallback('Frame', click_event)
while(cap.isOpened()):
    ret, frame = cap.read()
    if ret==True:
        frame= doprocess(frame)
        cv2.imshow('Frame',frame)
        #result.write(frame)
        if(cv2.waitKey(25)&0xFF == ord("q")):

            break
    else:
        break

cap.release()
#result.release()
cv2.destroyAllWindows()


import pytest

from draid.rule import (
        ObligationDeclaration, DataRuleContainer, AttributeCapsule,
        FlowRule, Propagate,
        )
from draid.rule import EqualAC

from draid.reason import prolog_handle

WhenImported = EqualAC('stage', 'import')

pc1 = AttributeCapsule.from_raw('name', [('str', 'UoE')])
pc1_2 = AttributeCapsule.from_raw('name', [
    ('str', 'UoE'),
    ('str', 'University of Earth')
    ])

pc2 = AttributeCapsule.from_raw('sens', [('str', '1')])

ob1 = (
        ObligationDeclaration('credit', [('name', 0)]),
        ObligationDeclaration('credit', [('name', 1)]),
        ObligationDeclaration('credit', [('name', 0)], WhenImported),
        ObligationDeclaration('credit', [('name', 1)], WhenImported),
        )
ob2 = ObligationDeclaration('hide', [('sens', 0)])

rule1 = DataRuleContainer([ob1[0], ob2], [pc1, pc2])
# rule2 = DataRuleContainer([ob1[0], ob1[1], ob2, ob3], [pc1, pc2])

flow_rule1 = FlowRule([
    Propagate('input1', 'output1'),
    Propagate('input1', 'output2'),
    Propagate('input2', 'output1'),
    ])

@pytest.mark.parametrize('drc, port', [
    (rule1, 'input1'),
    ])
def test_dump_data_rule(drc, port):
    d = prolog_handle.dump_data_rule(drc, port)
    assert d
    print(d)

@pytest.mark.parametrize('flow_rule', [
    flow_rule1,
    ])
def test_pl_act_flow_rule(flow_rule):
    d = prolog_handle.pl_act_flow_rule(flow_rule)
    assert d
    print(d)

@pytest.mark.parametrize('flow_rule', [
    flow_rule1,
    ])
def test_query_of_flow_rule(flow_rule):
    d = prolog_handle.query_of_flow_rule(flow_rule)
    assert d
    print(d)

@pytest.mark.parametrize('data_rules, flow_rule', [
    ({'input1': rule1}, flow_rule1),
    ])
def test_dispatch(data_rules, flow_rule):
    prolog_handle.PR_DIR = '/home/ryey/self/Edinburgh/PhD/exp/exp'
    prolog_handle.dispatch(data_rules, flow_rule)
    assert True


import cv2 as cv
import os
import numpy as np
from mtcnn.mtcnn import MTCNN

def detecting_faces():
    face_detector = MTCNN()
    capture_video = cv.VideoCapture(0)
    cropped_faces = []
    while True:
        frame_got, frame = capture_video.read()
        detected_faces = face_detector.detect_faces(frame)
        if detected_faces:
            for person in detected_faces:
                boxes = person['box']
                cv.rectangle(frame, (boxes[0], boxes[1]), (boxes[0] + boxes[2],
                                                           boxes[1] + boxes[3]), (0, 255, 0), 2)
                cropped_faces = frame[boxes[0] - 10:boxes[0] + boxes[2] + 50, boxes[1] - 10:boxes[1] + boxes[3] + 50]
        cv.imshow("Frame", frame)
        if cv.waitKey(1) & 0xFF == ord('q'):
            break

    capture_video.release()
    cv.destroyAllWindows()
    return cropped_faces

face_detected = detecting_faces()
print(face_detected)


import tensorflow as tf
import numpy as np
import utils
from utils import gs
import argparse
import os
import shutil
import json
import time

def name_var(x, name):
    return tf.identity(x, name=name)

def default_initializer(std=0.05):
    return tf.random_normal_initializer(0., std)

def squeeze(x, factor=2, backward=False):
    assert factor >= 1
    if factor == 1: return x
    shape = x.get_shape()
    height = int(shape[1])
    width = int(shape[2])
    n_channels = int(shape[3])
    if backward:
        assert n_channels >= 4 and n_channels % 4 == 0
        x = tf.reshape(x, (-1, height, width, int(n_channels / factor ** 2), factor, factor))
        x = tf.transpose(x, [0, 1, 4, 2, 5, 3])
        x = tf.reshape(x, (-1, int(height * factor), int(width * factor), int(n_channels / factor ** 2)))
    else:
        assert height % factor == 0 and width % factor == 0
        x = tf.reshape(x, [-1, height//factor, factor, width//factor, factor, n_channels])
        x = tf.transpose(x, [0, 1, 3, 5, 2, 4])
        x = tf.reshape(x, [-1, height//factor, width//factor, n_channels*factor*factor])
    return x

def actnorm(x, name, init=False, eps=1e-6, return_logdet=False, backward=False):
    def compute(x, logs, t):
        if backward:
            return (x * tf.exp(-logs)) - t
        else:
            return (x + t) * tf.exp(logs)
    def logdet(x, logs):
        h, w = x.get_shape().as_list()[1:3]
        val = tf.reduce_sum(logs) * h * w
        if backward:
            return -val
        else:
            return val
    with tf.variable_scope(name, reuse=(not init)):
        t = tf.get_variable("t", (1, 1, 1, gs(x)[-1]), trainable=True)
        logs = tf.get_variable("logs", (1, 1, 1, gs(x)[-1]), trainable=True)
        if init:
            x_mean, x_var = tf.nn.moments(x, axes=[0, 1, 2], keep_dims=True)
            logs_init = tf.log(1. / (tf.sqrt(x_var) + eps))
            t_init = -x_mean
            logsop = logs.assign(logs_init)
            top = t.assign(t_init)
            with tf.control_dependencies([logsop, top]):
                an = compute(x, logs_init, t_init)
        else:
            an = compute(x, logs, t)

        if return_logdet:
            return an, logdet(x, logs)
        else:
            return an

def NN(x, name, dim=128, init=False):
    def conv(h, d, k, name, init, nonlin=True):
        if nonlin:
            h = tf.layers.conv2d(h, d, (k, k), (1, 1), "same",
                                 name=name, use_bias=False,
                                 kernel_initializer=default_initializer())
            h = actnorm(h, name + "_an", init)
            h = tf.nn.relu(h)
            return h
        else:
            h = tf.layers.conv2d(h, d, (k, k), (1, 1), "same",
                                 name=name, use_bias=True,
                                 kernel_initializer=tf.zeros_initializer(),
                                 bias_initializer=tf.zeros_initializer())
            logs = tf.get_variable("logs", shape=[1, 1, 1, d], dtype=tf.float32, initializer=tf.zeros_initializer())
            s = tf.exp(logs)
            return h * s
    with tf.variable_scope(name, reuse=(not init)):
        nc = gs(x)[-1]
        h = conv(x, dim, 3, "h1", init)
        h = conv(h, dim, 1, "h2", init)
        h = conv(h, nc,  3, "h3", init, nonlin=False)
    return h

def split(x):
    nc = gs(x)[-1] // 2
    return x[:, :, :, :nc], x[:, :, :, nc:]

def combine(x1, x2):
    return tf.concat([x1, x2], axis=3)

def coupling_layer(in_feats, name, init=False, backward=False, eps=1e-6, dim=32):
    def get_vars(feats):
        logit_s = NN(feats, "logit_s", init=init, dim=dim) + 2.
        s = tf.sigmoid(logit_s) + eps
        t = NN(feats, "t", init=init, dim=dim)
        logdet = tf.reduce_sum(tf.log_sigmoid(logit_s), axis=[1, 2, 3])
        return s, t, logdet

    with tf.variable_scope(name):
        if backward:
            y = in_feats
            y1, y2 = split(y)
            s, t, logdet = get_vars(y1)
            x1 = y1
            x2 = (y2 / s) - t
            x = combine(x1, x2)
            return x, -logdet
        else:
            x = in_feats
            x1, x2 = split(x)
            s, t, logdet = get_vars(x1)
            y1 = x1
            y2 = (x2 + t) * s
            y = combine(y1, y2)
            return y, logdet

def random_rotation_matrix(nc):
    return np.linalg.qr(np.random.randn(nc, nc))[0].astype(np.float32)

def invconv_layer(in_feats, name, init=False, backward=False, eps=1e-6):
    conv = lambda f, k: tf.nn.conv2d(f, k, [1, 1, 1, 1], "SAME")
    with tf.variable_scope(name, reuse=(not init)):
        hh, ww, nc = gs(in_feats)[1:]
        w = tf.get_variable("w", dtype=tf.float32, initializer=random_rotation_matrix(nc))
        det = tf.matrix_determinant(w)
        logdet = tf.log(tf.abs(det) + eps) * hh * ww
        if backward:
            y = in_feats
            kernel = tf.reshape(tf.matrix_inverse(w), [1, 1, nc, nc])
            x = conv(y, kernel)
            return x, -logdet
        else:
            x = in_feats
            kernel = tf.reshape(w, [1, 1, nc, nc])
            y = conv(x, kernel)
            return y, logdet

def flow_step(in_feats, name, init=False, backward=False, dim=32):
    with tf.variable_scope(name, reuse=(not init)):
        if backward:
            y = in_feats
            x, logdet_coupling = coupling_layer(y, "coupling", init=init, backward=True, dim=dim)
            x, logdet_invconv = invconv_layer(x, "invconv", init=init, backward=True)
            x, logdet_actnorm = actnorm(x, "actnorm", init=init, return_logdet=True, backward=True)
            return x, logdet_actnorm + logdet_invconv + logdet_coupling
        else:
            x = in_feats
            x, logdet_actnorm = actnorm(x, "actnorm", init=init, return_logdet=True, backward=False)
            x, logdet_invconv = invconv_layer(x, "invconv", init=init, backward=False)
            y, logdet_coupling = coupling_layer(x, "coupling", init=init, backward=False, dim=dim)
            return y, logdet_actnorm + logdet_invconv + logdet_coupling

def scale_block(in_feats, name, flow_steps, init=False, backward=False, dim=32):
    with tf.variable_scope(name, reuse=(not init)):
        if backward:
            z = in_feats
            logdet = 0.
            for i in reversed(range(flow_steps)):
                z, step_logdet = flow_step(z, "step{}".format(i), init=init, backward=True, dim=dim)
                logdet += step_logdet
            x = squeeze(z, backward=True)
            return x, logdet
        else:
            x = in_feats
            z = squeeze(x, backward=False)
            logdet = 0.
            for i in range(flow_steps):
                z, step_logdet = flow_step(z, "step{}".format(i), init=init, backward=False, dim=dim)
                logdet += step_logdet
            return z, logdet

def net(in_feats, name, num_blocks, flow_steps, init=False, backward=False, width=32):
    with tf.variable_scope(name, reuse=(not init)):
        if backward:
            zs = in_feats
            logdet = 0.
            z = zs[-1]
            for i in reversed(range(num_blocks)):
                z, block_logdet = scale_block(z, "block{}".format(i), flow_steps, init=init, backward=True, dim=width)
                logdet += block_logdet
                if i > 0:
                    zi = zs[i - 1]
                    z = combine(z, zi)
            return z, logdet
        else:
            z = in_feats
            logdet = 0.
            zs = []
            for i in range(num_blocks):
                z, block_logdet = scale_block(z, "block{}".format(i), flow_steps, init=init, backward=False, dim=width)
                logdet += block_logdet
                if i == num_blocks - 1:
                    zs.append(z)
                else:
                    z, zi = split(z)
                    zs.append(zi)
            return zs, logdet

def logpx(zs, logdet):
    logpz = tf.add_n([tf.reduce_sum(utils.normal_logpdf(z, 0., 0.), axis=[1, 2, 3]) for z in zs])
    ave_logpz = tf.reduce_mean(logpz)
    ave_logdet = tf.reduce_mean(logdet)
    total_logprob = (ave_logpz + ave_logdet)
    tf.summary.scalar("logdet", ave_logdet)
    tf.summary.scalar("logp", ave_logpz)
    return total_logprob

def create_experiment_directory(args):
    # write params
    with open(os.path.join(args.train_dir, "params.txt"), 'w') as f:
        f.write(json.dumps(args.__dict__))
    # copy code
    code_dest_dir = os.path.join(args.train_dir, "code")
    os.mkdir(code_dest_dir)
    code_dir = os.path.dirname(__file__)
    code_dir = '.' if code_dir == '' else code_dir
    python_files = [os.path.join(code_dir, fn) for fn in os.listdir(code_dir) if fn.endswith(".py")]
    for pyf in python_files:
        print(pyf, code_dest_dir)
        shutil.copy2(pyf, code_dest_dir)
    os.mkdir(os.path.join(args.train_dir, "best"))
    os.mkdir(os.path.join(args.train_dir, "backup"))

def get_lr(epoch, args):
    epoch_lr = (args.lr * (epoch + 1) / args.epochs_warmup) if epoch < args.epochs_warmup + 1 else args.lr
    # get decayed lr
    if args.lr_scalemode == 0:
        return epoch_lr
    else:
        lr_scale = args.decay_factor ** (epoch // args.epochs_decay)
        epoch_lr *= lr_scale
        return epoch_lr

if __name__ == "__main__":
    parser = argparse.ArgumentParser()
    parser.add_argument("--train_dir", type=str, default="/tmp/train")
    parser.add_argument("--dataset", type=str, default='mnist', help="Problem (mnist/cifar10/svhn)")
    parser.add_argument("--num_valid", type=int, default=None,
                        help="The number of examples to place into the validaiton set (only for svhn and cifar10)")
    parser.add_argument("--num_labels", type=int, default=None, help="Number of labeled examples to use")
    parser.add_argument("--load_path", type=str, default=None, help="Path for load saved checkpoint from")
    parser.add_argument("--log_iters", type=int, default=100, help="iters per each print and summary")

    # Optimization hyperparams:
    parser.add_argument("--epochs", type=int, default=100000, help="Train epoch size")
    parser.add_argument("--batch_size", type=int, default=64, help="batch size")
    parser.add_argument("--init_batch_size", type=int, default=1024, help="batch size for init")
    parser.add_argument("--lr", type=float, default=0.001, help="Base learning rate")
    parser.add_argument("--lr_scalemode", type=int, default=0, help="Type of learning rate scaling. 0=none, 1=step.")
    parser.add_argument("--epochs_warmup", type=int, default=10, help="Warmup epochs")
    parser.add_argument("--epochs_valid", type=int, default=1, help="Epochs between valid")
    parser.add_argument("--epochs_backup", type=int, default=10, help="Epochs between backup saving")
    parser.add_argument("--epochs_decay", type=int, default=250, help="Epochs between lr decay")
    parser.add_argument("--decay_factor", type=float, default=.1, help="Multiplier on learning rate")

    # Model hyperparams:
    parser.add_argument("--width", type=int, default=128, help="Width of hidden layers")
    parser.add_argument("--depth", type=int, default=8, help="Depth of network")
    parser.add_argument("--n_levels", type=int, default=3, help="Number of levels")
    parser.add_argument("--n_bits_x", type=int, default=5, help="Number of bits of x")

    # Finetuning arguments
    parser.add_argument("--finetune", type=int, default=0, help="if 0, then train generaitve, 1 then finetune")
    parser.add_argument("--clf_type", type=str, default="unwrap")

    args = parser.parse_args()
    args.n_bins_x = 2.**args.n_bits_x
    assert args.finetune in (0, 1)
    assert args.clf_type in ("unwrap", "pool")
    assert not os.path.exists(args.train_dir), "This directory already exists..."
    train_writer = tf.summary.FileWriter(os.path.join(args.train_dir, "train"))
    test_writer = tf.summary.FileWriter(os.path.join(args.train_dir, "test"))
    valid_writer = tf.summary.FileWriter(os.path.join(args.train_dir, "valid"))
    # setup experiment directory, copy current version of the code, save parameters
    create_experiment_directory(args)

    # create session
    sess = tf.Session()

    dataset_fn = {'svhn': utils.SVHNDataset, 'cifar10': utils.CIFAR10Dataset, 'mnist': utils.MNISTDataset}[args.dataset]
    dataset = dataset_fn(
        args.batch_size,
        init_size=args.init_batch_size, n_labels=args.num_labels, n_valid=args.num_valid, n_bits_x=args.n_bits_x
    )

    # unpack labeled examples
    x, y = dataset.x, tf.to_int64(dataset.y)

    # build graph
    z_init, _ = net(x, "net", args.n_levels, args.depth, width=args.width, init=True)
    z, logdet = net(x, "net", args.n_levels, args.depth, width=args.width)

    # train to optimize logp(x)
    if args.finetune == 0:
        print("Training for generation")
        # input reconstructions
        x_recons, _ = net(z, "net", args.n_levels, args.depth, width=args.width, backward=True)
        # samples
        z_samp = [tf.random_normal(tf.shape(_z)) for _z in z]
        x_samp, _ = net(z_samp, "net", args.n_levels, args.depth, width=args.width, backward=True)

        # compute objective logp(x)
        logpz = tf.add_n([tf.reduce_sum(utils.normal_logpdf(z_l, 0., 0.), axis=[1, 2, 3]) for z_l in z])
        logpx = logpz + logdet
        objective = tf.reduce_mean(logpx) - np.log(args.n_bins_x) * np.prod(gs(x)[1:])
        loss = -objective

        # create optimizer
        lr = tf.placeholder(tf.float32, [], name="lr")
        optim = tf.train.AdamOptimizer(lr)
        opt = optim.minimize(loss)

        # summaries and visualizations
        x_recons = utils.postprocess(x_recons, n_bits_x=args.n_bits_x)
        x_samp = utils.postprocess(x_samp, n_bits_x=args.n_bits_x)
        recons_error = tf.reduce_mean(tf.square(tf.to_float(utils.postprocess(x, n_bits_x=args.n_bits_x) - x_recons)))
        tf.summary.image("x_sample", x_samp)
        tf.summary.image("x_recons", x_recons)
        tf.summary.image("x", x)
        tf.summary.scalar("recons", recons_error)
        tf.summary.scalar("lr", lr)
        # summaries for test datasets
        loss_summary = tf.summary.scalar("loss", loss)
        m_logdet = tf.reduce_mean(logdet)
        m_logpz = tf.reduce_mean(logpz)
        logdet_summary = tf.summary.scalar("logdet", m_logdet)
        logpz_summary = tf.summary.scalar("logpz", m_logpz)
        # the summary op to call for the training data
        train_summary = tf.summary.merge_all()
        # get all summaries we want to display on the test set
        test_summaries = [loss_summary, logdet_summary, logpz_summary]
        # get the values we need to call to get mean stats
        test_values = [loss, m_logdet, m_logpz]
        test_value_names = ['loss', 'logdet', 'logpz']
        test_summary = tf.summary.merge(test_summaries)

        # initialize variables
        sess.run(tf.global_variables_initializer())
        # initialize act-norm parameters
        sess.run(dataset.use_init)
        sess.run(z_init)

        # create savers for the best validation models and regularly
        best_saver = tf.train.Saver(max_to_keep=5)
        backup_saver = tf.train.Saver(max_to_keep=5)

        # restore model if asked
        if args.load_path is not None:
            print("restoring model...")
            backup_saver.restore(sess, args.load_path)
            start_epoch = int(args.load_path.split('-')[-1])
        else:
            start_epoch = 0

        # evaluation code block
        def evaluate(init_op, writer, name):
            sess.run(init_op)
            summary_values = []
            while True:
                try:
                    summary_values.append(sess.run(test_values))
                except tf.errors.OutOfRangeError:
                    summary_values = np.array(summary_values).mean(axis=0)
                    print("{}: ...".format(name))
                    for val_name, val_val in zip(test_value_names, summary_values):
                        print("    {}: {}".format(val_name, val_val))
                    fd = {node: val for node, val in zip(test_values, summary_values)}
                    sstr = sess.run(test_summary, feed_dict=fd)
                    writer.add_summary(sstr, cur_iter)
                    # return loss to determine best model
                    return summary_values[0]

        # training loop
        cur_iter = 0
        best_valid = np.inf
        for epoch in range(start_epoch, args.epochs):
            sess.run(dataset.use_train)
            t_start = time.time()
            # get lr for this epoch
            epoch_lr = get_lr(epoch, args)
            while True:
                try:
                    if cur_iter % args.log_iters == 0:
                        _re, _l, _, sstr = sess.run([recons_error, loss, opt, train_summary], feed_dict={lr: epoch_lr})
                        train_writer.add_summary(sstr, cur_iter)
                        print(cur_iter, _l, _re)

                    else:
                        _ = sess.run(opt, feed_dict={lr: epoch_lr})

                    cur_iter += 1
                except tf.errors.OutOfRangeError:
                    print("Completed epoch {} in {}".format(epoch, time.time() - t_start))
                    break

            # get accuracy on test set
            if epoch % args.epochs_valid == 0:
                evaluate(dataset.use_test, test_writer, "Test")
                # if we have a validation set, get validation accuracy
                if dataset.use_valid is not None:
                    valid_loss = evaluate(dataset.use_valid, valid_writer, "Valid")
                    if valid_loss < best_valid:
                        print("Best performing model with loss: {}".format(valid_loss))
                        best_valid = valid_loss
                        best_saver.save(sess, "{}/best/model.ckpt".format(args.train_dir), global_step=epoch)

            # backup model
            if epoch % args.epochs_backup == 0:
                backup_saver.save(sess, "{}/backup/model.ckpt".format(args.train_dir), global_step=epoch)

    # finetune classification
    else:
        print("Finetuning model for classification")
        # create saver for restoring the weights trained for generation
        pretrain_saver = tf.train.Saver()

        # get top z as features for classification layer
        top_z = z[-1]
        if args.clf_type == "unwrap":
            zsize = np.prod(gs(top_z)[1:])
            feats = tf.reshape(top_z, [-1, zsize])
        elif args.clf_type == "pool":
            feats = tf.reduce_mean(top_z, axis=[1, 2])

        logits = tf.layers.dense(feats, dataset.n_class, name="logits")
        loss = tf.reduce_mean(tf.nn.sparse_softmax_cross_entropy_with_logits(labels=y, logits=logits))
        preds = tf.argmax(logits, axis=1)
        correct = tf.to_float(tf.equal(preds, y))
        accuracy = tf.reduce_mean(correct)

        # create optimizer
        lr = tf.placeholder(tf.float32, [], name="lr")
        optim = tf.train.AdamOptimizer(lr)
        opt = optim.minimize(loss)

        # make summaries
        tf.summary.image("x", x)
        tf.summary.scalar("lr", lr)
        # summaries for test datasets
        loss_summary = tf.summary.scalar("loss", loss)
        acc_summary = tf.summary.scalar("accuracy", accuracy)
        # the summary op to call for the training data
        train_summary = tf.summary.merge_all()
        # get all summaries we want to display on the test set
        test_summaries = [loss_summary, acc_summary]
        # get the values we need to call to get mean stats
        test_values = [loss, accuracy]
        test_value_names = ['loss', 'acc']
        test_summary = tf.summary.merge(test_summaries)

        # initialize variables
        sess.run(tf.global_variables_initializer())
        # initialize act-norm parameters
        sess.run(dataset.use_init)
        sess.run(z_init)

        # create savers for the best validation models and regularly
        best_saver = tf.train.Saver(max_to_keep=5)
        backup_saver = tf.train.Saver(max_to_keep=5)
        # restore pretrained weights
        if args.load_path is not None:
            print("Loading pretrained weights")
            pretrain_saver.restore(sess, args.load_path)

        # evaluation code block
        def evaluate(init_op, writer, name):
            sess.run(init_op)
            summary_values = []
            while True:
                try:
                    summary_values.append(sess.run(test_values))
                except tf.errors.OutOfRangeError:
                    summary_values = np.array(summary_values).mean(axis=0)
                    print("{}: ...".format(name))
                    for val_name, val_val in zip(test_value_names, summary_values):
                        print("    {}: {}".format(val_name, val_val))
                    fd = {node: val for node, val in zip(test_values, summary_values)}
                    sstr = sess.run(test_summary, feed_dict=fd)
                    writer.add_summary(sstr, cur_iter)
                    # return accuracy to determine best model
                    return summary_values[1]

        # training loop
        cur_iter = 0
        best_valid = 0.
        for epoch in range(args.epochs):
            sess.run(dataset.use_train)
            t_start = time.time()
            # get lr for this epoch
            epoch_lr = get_lr(epoch, args)
            while True:
                try:
                    if cur_iter % args.log_iters == 0:
                        _l, _a, _, sstr = sess.run([loss, accuracy, opt, train_summary],
                                                    feed_dict={lr: epoch_lr})
                        train_writer.add_summary(sstr, cur_iter)
                        print(cur_iter, _l, _a)

                    else:
                        _ = sess.run(opt, feed_dict={lr: epoch_lr})

                    cur_iter += 1
                except tf.errors.OutOfRangeError:
                    print("Completed epoch {} in {}".format(epoch, time.time() - t_start))
                    break

            # get accuracy on test set
            if epoch % args.epochs_valid == 0:
                evaluate(dataset.use_test, test_writer, "Test")
                # if we have a validation set, get validation accuracy
                if dataset.use_valid is not None:
                    valid_loss = evaluate(dataset.use_valid, valid_writer, "Valid")
                    if valid_loss > best_valid:
                        print("Best performing model with accuracy: {}".format(valid_loss))
                        best_valid = valid_loss
                        best_saver.save(sess, "{}/best/model.ckpt".format(args.train_dir), global_step=epoch)

            # backup model
            if epoch % args.epochs_backup == 0:
                backup_saver.save(sess, "{}/backup/model.ckpt".format(args.train_dir), global_step=epoch)


import argparse
import random
import torch.nn as nn
import torch
from transformers import AutoTokenizer, AutoModel, AutoModelForMultipleChoice
from torch.utils.data import DataLoader, Dataset, SubsetRandomSampler
from torch.optim import AdamW
from torch.nn import CrossEntropyLoss
from tqdm import tqdm
from sklearn.metrics import accuracy_score
import numpy as np
import ltn
import logging
from torch.optim import lr_scheduler
from model import SequenceModel, LogitsToPredicate
from dataset import dataset_creator
import os
import sklearn.metrics as metrics
import pprint
import ltn
import wandb
import math
from factory import get_constants

os.environ["TOKENIZERS_PARALLELISM"] = "True"
device = 'cuda:0' if torch.cuda.is_available() else 'cpu'

number_of_labels = {
    'RTE': 2,
    'MNLI': 3,
    'QNLI': 2,
    'COLA': 2,
    'SST': 2,
    'WNLI': 2
    }

task_to_metric = {
    'RTE':  metrics.accuracy_score,
    'MNLI': metrics.accuracy_score,
    'QNLI': metrics.accuracy_score,
    'COLA': metrics.matthews_corrcoef,
    'SST': metrics.accuracy_score,
    'WNLI': metrics.accuracy_score
    }

def parse_args():
    parser = argparse.ArgumentParser(description="ArgumentParser for GLUE scripts")
    parser.add_argument("--batch_size", type=int, default=32, help="The batch size to use during training.")
    parser.add_argument("--weight_decay", type=float, default=0.01, help="The weight decay to use during training.")
    parser.add_argument("--model_name", type=str, default="bert-base-uncased", help="The pretrained model to use")
    parser.add_argument("--task", type=str, default="rte", help="The task to train on")
    parser.add_argument("--optimizer", type=str, default="adamw", help="The optimizer function to train with")
    parser.add_argument("--scheduler", type=str, default="cosine", help="The scheduler to train with")
    parser.add_argument("--epochs", type=int, default=3, help="The number of epochs.")
    parser.add_argument("--p_value", type=int, default=2, help="The p value for pMeanAgg")
    parser.add_argument("--debug", action='store_true', help="Trigger debug mode")
    parser.add_argument("--logic_mode", action='store_true', help="Trigger the use of LTN and SatAgg objective")
    parser.add_argument("--freeze", action='store_true', help="Only fine tune classification head")
    parser.add_argument("--test", action='store_true', help="Trigger test eval")
    parser.add_argument("--lr", type=float, default=3e-5, help="The learning rate).")
    parser.add_argument("--seed", type=int, default=42, help="The rng seed")
    parser.add_argument("--gradient_clip", action='store_true', help="The gradient clip")
    parser.add_argument("--beta", type=float, default=1, help="The adam momentum")
    parser.add_argument("--patience", type=int, default=2, help="The patience value")
    parser.add_argument("--dropout", type=float, default=0.1, help="The dropout value")
    parser.add_argument("--sample", type=float, default=1.0, help="The percentage of the original dataset to use for training")
    parser.add_argument("--max_steps", default=31250, type=int, help="Total number of training steps to perform.")
    parser.add_argument("--warmup_proportion", default=0.01, type=float, help="Proportion of training to perform linear learning rate warmup for. E.g., 0.1 = 10%% of training.")

    args = parser.parse_args()
    return args

def main(args):
    print("====================================================================================================")
    logging.info(f"Initialised training on task: {args.task.upper()}, debug={args.debug}, device={device}")
    print("====================================================================================================")
    print("\n")

    config = {
            "task": args.task,
            "learning_rate": args.lr,
            "weight_decay": args.weight_decay,
            "epochs": args.epochs,
            "batch_size": args.batch_size,
            "scheduler": args.scheduler,
            "optimizer": args.optimizer,
            "seed": args.seed,
            "model": args.model_name,
            "freeze": args.freeze,
            "test": args.test,
            "dropout": args.dropout,
            "logic_mode": args.logic_mode,
            "p_value": args.p_value,
            "sample": args.sample
        }

    if not args.debug:
        wandb.init(project="ltn", config=config, entity="sondrewo")

    logging.info("HYPERPARAMETERS: \n")
    pp = pprint.PrettyPrinter(indent=4)
    pp.pprint(config)
    print("\n")

    train_dataset = dataset_creator(args.task)(tokenizer_path_or_name=args.model_name, root_data_path="./data", split='train', max_length=64)
    val_dataset = dataset_creator(args.task)(tokenizer_path_or_name=args.model_name, root_data_path="./data", split='dev', max_length=64)

    if args.sample < 1.:
        logging.info(f"Using downsampled train data, with rate {args.sample}")
    train_length = len(train_dataset)
    indices = torch.randperm(len(train_dataset))[:int(len(train_dataset)*args.sample)]

    train_loader = DataLoader(train_dataset, sampler=SubsetRandomSampler(indices), batch_size=args.batch_size, num_workers=1, drop_last=True)
    val_loader = DataLoader(val_dataset, batch_size=args.batch_size, shuffle=False, num_workers=1, drop_last=True)

    logging.info(train_dataset.get_decoded_example(13))

    model = SequenceModel(device, args.model_name, number_of_labels[args.task.upper()], args.dropout).to(device)

    if args.logic_mode:
        P = ltn.Predicate(LogitsToPredicate(device, model)).to(device)
        Forall = ltn.Quantifier(ltn.fuzzy_ops.AggregPMeanError(p=args.p_value), quantifier="f")
        SatAgg = ltn.fuzzy_ops.SatAgg()
        constants = get_constants(args.task, device)

    params = list(model.named_parameters())
    no_decay = ['bias', 'layer_norm', 'embedding']
    decay_params = [(n, p) for n, p in params if not any(nd in n for nd in no_decay)]
    no_decay_params = [(n, p) for n, p in params if any(nd in n for nd in no_decay)]
    optimizer_grouped_parameters = [
        {'params': [p for _, p in decay_params], 'weight_decay': args.weight_decay},
        {'params': [p for _, p in no_decay_params], 'weight_decay': 0.0}
    ]

    if args.optimizer == "adam" or args.optimizer == "adamw":
        optimizer = torch.optim.AdamW(
            optimizer_grouped_parameters,
            lr=args.lr,
            betas=(0.9, 0.98),
            eps=1e-6,
        )
    else:
        pass

    max_steps = len(train_loader) * args.epochs

    if args.scheduler == "cosine":
        def cosine_schedule_with_warmup(optimizer, num_warmup_steps: int, num_training_steps: int, min_factor: float):
            def lr_lambda(current_step):
                if current_step < num_warmup_steps:
                    return float(current_step) / float(max(1, num_warmup_steps))
                progress = float(current_step - num_warmup_steps) / float(max(1, num_training_steps - num_warmup_steps))
                lr = max(min_factor, min_factor + (1 - min_factor) * 0.5 * (1.0 + math.cos(math.pi * progress)))

                return lr

            return torch.optim.lr_scheduler.LambdaLR(optimizer, lr_lambda)

        scheduler = cosine_schedule_with_warmup(optimizer, int(max_steps * args.warmup_proportion), max_steps, 0.1)

    elif args.scheduler == "linear":
        scheduler = lr_scheduler.ChainedScheduler([
            lr_scheduler.LinearLR(optimizer, start_factor=1e-9, end_factor=1.0, total_iters=int(max_steps * args.warmup_proportion)),
            lr_scheduler.LinearLR(optimizer, start_factor=1.0, end_factor=1e-9, total_iters=max_steps)
        ])

    criterion = CrossEntropyLoss()

    if args.freeze:
        for param in model.encoder.parameters():
            param.requires_grad = False

    for epoch in range(args.epochs):
        logging.info(f"Started training at epoch {epoch}")
        model.train()
        train_loss = 0.
        running_losses = []
        mean_sat = 0.
        for i, (input_ids, attention_masks, y) in enumerate(tqdm(train_loader)):
            optimizer.zero_grad()
            input_ids, attention_masks, y = input_ids.to(device), attention_masks.to(device), y.to(device)
            y = y.squeeze(-1)
            data = torch.cat((input_ids, attention_masks), -1).to(device)
            if args.logic_mode:
                if number_of_labels[args.task.upper()] == 3:
                    x_A = ltn.Variable("x_A", data[y == 0]) # class A examples
                    x_B = ltn.Variable("x_B", data[y == 1]) # class B examples
                    x_C = ltn.Variable("x_B", data[y == 2]) # class C examples
                    satt_agg = SatAgg(
                        Forall(x_A, P(x_A, constants[0])),
                        Forall(x_B, P(x_B, constants[1])),
                        Forall(x_B, P(x_B, constants[2]))
                    )
                elif number_of_labels[args.task.upper()] == 2:
                    x_A = ltn.Variable("x_A", data[y == 0]) # class A examples
                    x_B = ltn.Variable("x_B", data[y == 1]) # class B examples
                    satt_agg = SatAgg(
                        Forall(x_A, P(x_A, constants[0])),
                        Forall(x_B, P(x_B, constants[1])),
                    )
                mean_sat += satt_agg
                loss = 1. - satt_agg
                train_loss += loss.item()
            else:
                out = model(data)
                loss = criterion(out, y)
            c = loss.item()
            train_loss += c
            running_losses.append(c)
            loss.backward()
            optimizer.step()
            scheduler.step()
            if args.debug:
                break
            else:
                wandb.log({"running_loss": np.mean(running_losses)})

        model.eval()
        with torch.no_grad():
            val_loss = 0.
            mean_sat = 0.
            golds = []
            preds = []
            for i, (input_ids, attention_masks, y) in enumerate(tqdm(val_loader)):
                input_ids, attention_masks, y = input_ids.to(device), attention_masks.to(device), y.to(device)
                y = y.squeeze(-1)
                data = torch.cat((input_ids, attention_masks), -1).to(device)
                if args.logic_mode:
                    if number_of_labels[args.task.upper()] == 3:
                        x_A = ltn.Variable("x_A", data[y == 0]) # class A examples
                        x_B = ltn.Variable("x_B", data[y == 1]) # class B examples
                        x_C = ltn.Variable("x_C", data[y == 2]) # class C examples
                        satt_agg = SatAgg(
                            Forall(x_A, P(x_A, constants[0])),
                            Forall(x_B, P(x_B, constants[1])),
                            Forall(x_B, P(x_C, constants[2]))
                        )

                    elif number_of_labels[args.task.upper()] == 2:
                        x_A = ltn.Variable("x_A", data[y == 0]) # class A examples
                        x_B = ltn.Variable("x_B", data[y == 1]) # class B examples
                        satt_agg = SatAgg(
                            Forall(x_A, P(x_A, constants[0])),
                            Forall(x_B, P(x_B, constants[1])),
                        )
                    mean_sat += satt_agg
                    loss = 1. - satt_agg
                out = model(data)
                if not args.logic_mode:
                    loss = criterion(out, y)
                val_loss += loss.item()
                golds.extend(y.cpu().tolist())
                y_hat = torch.argmax(out.cpu(), dim=-1)
                preds.extend(y_hat.tolist())

            score = task_to_metric[args.task.upper()](golds, preds)

        t_l = train_loss / len(train_loader)
        v_l = val_loss / len(val_loader)

        if not args.debug:
            wandb.log({"train_loss_epoch": t_l})
            wandb.log({"val_loss_epoch": v_l})
            wandb.log({"score": score})

            if args.logic_mode:
                wandb.log({"sat_level_epoch": mean_sat / len(val_loader)})

        if args.logic_mode:
            logging.info(f"LOSS - train: {(t_l):.3f}, valid: {(v_l):.3f}, SAT - val: {mean_sat / len(val_loader):.3f}, SCORE - val: {score:.3f} ")
        else:
            logging.info(f"LOSS - train: {(t_l):.3f}, valid: {(v_l):.3f}, METRIC SCORE - val: {score:.3f} ")

if __name__ == "__main__":
    args = parse_args()
    logging.basicConfig(
        format="%(asctime)s - %(levelname)s - %(name)s -   %(message)s",
        datefmt="%m/%d/%Y %H:%M:%S",
        level=logging.INFO,
    )
    torch.manual_seed(args.seed)
    torch.cuda.manual_seed(args.seed)
    torch.cuda.manual_seed_all(args.seed)
    np.random.seed(args.seed)
    random.seed(args.seed)
    torch.backends.cudnn.benchmark = False
    torch.backends.cudnn.deterministic = True
    main(args)


"""
The sea sub compiler entry point.
"""
import os

import argparse
import pathlib

from seasub import seasub

_DEFAULT_OPTIMIZATION_LEVEL = 1

def main():
    optimization_levels = {0: "No optimization", 1: "Constant folding"}
    optimization_level_help = "\n".join(f"\t{level}: {description}"
                                        for level, description in optimization_levels.items())
    parser = argparse.ArgumentParser(description="A compiler for the Sea Sub (C subset) language.",
                                     formatter_class=argparse.RawTextHelpFormatter)
    parser.add_argument('input', type=pathlib.Path,
                        help="the c source file to be compiled, output is stored in the same location with '.s' ending")
    parser.add_argument('-o',
                        help=f"optimization level (default {_DEFAULT_OPTIMIZATION_LEVEL})\n{optimization_level_help}",
                        dest='optimization_level', type=int, metavar='N', default=_DEFAULT_OPTIMIZATION_LEVEL)
    parser.add_argument('--ast', type=pathlib.Path, metavar='ast.dot',
                        help=".dot file to store the abstract syntax tree")
    parser.add_argument('--symbol-table', type=pathlib.Path, metavar='symbol-table.dot',
                        help=".dot file to store the symbol table")
    parser.add_argument('--save-intermediate-code', action='store_true', help="save the intermediate code")
    args = parser.parse_args()
    seasub.run(args.input, f'{os.path.splitext(args.input)[0]}.s', args.optimization_level,
               ast_graph_path=args.ast,
               symbol_table_graph_path=args.symbol_table,
               save_intermediate_code=args.save_intermediate_code)

if __name__ == "__main__":
    main()


#! /usr/bin/env python3

import psycopg2
from config import config

device = "01"
timestamp = "06/01/19 00:00:00"
state = 100

def insert_data(device, timestamp, state):
    """insert a new row into the table"""
    sql = """INSERT INTO statedata(device, datetime, state)
        VALUES(%s,%s,%s) RETURNING datapoint;"""
    conn = None
    datapoint = None
    try:
        # read datebase configuration
        params = config()
        # connect to the PostgreSQL database
        conn = psycopg2.connect(**params)
        # create a new cursor
        cur = conn.cursor()
        # execute the INSERT statement
        cur.execute(sql,(device,timestamp,state,))
        # get the generated id back
        datapoint = cur.fetchone()[0]
        # commit the changes to the database
        conn.commit()
        # close communication with the database
        cur.close()
    except (Exception,psycopg2.DatabaseError) as error:
        print(error)
    finally:
        if conn is not None:
            conn.close()

    return datapoint

if __name__ == '__main__':
    # insert one row
    insert_data(device, timestamp, state)


import logging

class ColoredFormatter(logging.Formatter):
    color_map = {
        logging.CRITICAL: '\033[91m',  # red
        logging.ERROR: '\033[91m',  # red
        logging.WARNING: '\033[93m',  # yellow
        logging.INFO: '\033[92m',  # green
        logging.DEBUG: '\033[94m',  # blue
    }
    reset_seq = '\033[0m'

    def format(self, record: logging.LogRecord) -> str:
        msg = super().format(record)
        return self.color_map.get(record.levelno, '') + msg + self.reset_seq

def get_logger(level=logging.ERROR):
    logger = logging.getLogger(__name__)
    logger.setLevel(level)
    ch = logging.StreamHandler()
    ch.setLevel(level)
    fmt = ColoredFormatter()
    ch.setFormatter(fmt)
    logger.addHandler(ch)
    return logger

logger = get_logger(logging.DEBUG)
logging.captureWarnings(True)
logger.debug('init logger')


from django.urls import path

from .views import (
    MainShareHoldersView, CorporationInfoView,
    CorporationSearchView ,IncomeStatementView,
    CorpExcelExporter , ConglomerateListView
)

urlpatterns = [
    path('', CorporationInfoView.as_view()),
    path('/search', CorporationSearchView.as_view()),
    path('/main-shareholders', MainShareHoldersView.as_view()),
    path('/income-statement', IncomeStatementView.as_view()),
    path('/excels', CorpExcelExporter.as_view()),
    path('/conglomerate', ConglomerateListView.as_view())
]


import csv
import os
from smtplib import SMTPException

from django.conf import settings
from django.core.mail import EmailMessage
from django.core.management.base import BaseCommand
from django.utils import timezone

from matricula.models import ProcesoMatricula, MatriculaEstudiante

class Command(BaseCommand):
    help = '''
    Genera el reporte de matrícula DEMRE del proceso de matrícula activo y lo envía al correo
    especificado.
    '''

    def add_arguments(self, parser):
        parser.add_argument(
            'correos', metavar='correo', type=str, default=[],
            nargs='*',
            help='opcional: correos a los que enviar el reporte generado'
        )

    def handle(self, *args, **options):
        generar_reporte(correos=options['correos'])
        return

DIR_REPORTES = 'datos_estudiantes/reportes_matricula'

TIPOS_IDENTIFICACION = {
    0: 'C',
    1: 'P',
}

TIPOS_MATRICULAS = {
    1: 1,  # regular
    5: 2,  # repostulacion
    -1: 3,  # oficio de verificacion
    4: 4,  # BEA
    -2: 5,  # verificacion por acuerdo
    3: 6,  # PACE
    2: 7,  # admisión especial
    6: -1,  # trans externa
    7: -2,  # trans interna
}

def generar_reporte(correos):
    # asegurar que exista la carpeta
    if not os.path.exists(DIR_REPORTES):
        os.makedirs(DIR_REPORTES)

    hoy = timezone.localtime().strftime("%Y-%m-%d_%H-%M")
    nombre_archivo = f'{DIR_REPORTES}/reporte_{hoy}.csv'

    with open(nombre_archivo, 'w', newline='') as archivo_reporte:
        writer = csv.writer(archivo_reporte, delimiter=';')

        # insertar header
        writer.writerow([
            'TIPO_IDENTIFICACION',
            'NRO_IDENTIFICACION',
            'DIGITO_VERIFICADOR',
            'CODIGO_CARRERA',
            'TIPO_MATRICULA',
            'FECHA_MATRICULA',
            'HORA_MATRICULA',
            # 'VIA_INGRESO',
        ])

        proceso_activo = ProcesoMatricula.objects.get(activo=True)
        matriculas = MatriculaEstudiante.objects.filter(
            proceso_matricula=proceso_activo,
            estado=1,  # valido
            estudiante__periodo_ingreso=proceso_activo.periodo_ingreso,
        ).prefetch_related(
            'estudiante__persona',
            'estudiante__plan',
            'orden_compra',
        )
        matriculas_fecha = matriculas.order_by('orden_compra__fecha_pago')

        for matricula in matriculas_fecha:
            fecha_pago = timezone.localtime(matricula.orden_compra.fecha_pago)
            if not fecha_pago:
                print(
                    f'fecha de pago no definida, utilizando fecha de creación para '
                    f'la orden de compra {matricula.orden_compra.id}'
                )
                fecha_pago = matricula.orden_compra.creado
            if matricula.estudiante.via_ingreso_id != 6:
                writer.writerow([
                    TIPOS_IDENTIFICACION[matricula.estudiante.persona.tipo_identificacion],
                    matricula.estudiante.persona.numero_documento,
                    matricula.estudiante.persona.digito_verificador,
                    matricula.estudiante.plan.codigo_demre,
                    TIPOS_MATRICULAS[matricula.estudiante.via_ingreso.id],
                    fecha_pago.strftime("%d/%m/%Y"),
                    fecha_pago.strftime("%H:%M"),
                    # matricula.estudiante.via_ingreso.nombre,
                ])

    print(f'reporte generado y almacenado en {nombre_archivo}')

    if correos:
        print(f'enviando a los correos: {", ".join(correos)}')

        email = EmailMessage(
            subject=f'Reporte matrícula DEMRE {hoy}',
            body='',
            to=correos,
            from_email=settings.EMAIL_HOST_USER,
            reply_to=[settings.EMAIL_HOST_USER],
        )
        email.attach_file(nombre_archivo)

        try:
            email.send()
        except SMTPException:
            print('ha habido un error al intentar enviar el mail.')

    return


# train_test_fastai_models.py #
from fastai.vision.learner import *
from fastai.basic_train import *
from fastai.callbacks import *
from fastai.vision import *
from fastai import *
import fastai
import torch.nn as nn
import torch
import sklearn.metrics as skmetrics
import itertools
import copy
import PIL
from PIL import Image
from sklearn.model_selection import train_test_split
import random
import pickle
import cv2
import matplotlib.pyplot as plt
import pandas as pd
import numpy as np
import time
import sys
import os
import warnings
warnings.filterwarnings('always')
warnings.filterwarnings('ignore')

def training(subset):
    train_df = pd.read_csv("./data/"+subset+"train.csv",
                           usecols=["file", "label"])
    # valid_df = pd.read_csv("./data/"+subset+"_valid.csv",
    #                        usecols=["file", "label"])
    train_valid_df = train_df
    ds_tfms = get_transforms(
        do_flip=False, flip_vert=False, max_rotate=30, max_zoom=1.1, max_warp=0.3)
    # ImageDataBunch.from_folder seems to be dysfunctional
    data = ImageDataBunch.from_df(
        "", train_valid_df, ds_tfms=ds_tfms, size=28, bs=1024)
    print(data)
    learn = cnn_learner(data, models.resnet18, metrics=accuracy)
    learn.lr_find(start_lr=1e-6, end_lr=1e1, stop_div=True, num_it=200)

    plt.figure()
    learn.recorder.plot(suggestion=True)
    plt.title("Optimal Learning Rate - "+subset)
    plt.savefig("./graphics/"+subset+"_lr_selection.png")
    plt.savefig("./graphics/"+subset+"_lr_selection.pdf")
    plt.close()

    lr = learn.recorder.min_grad_lr
    print(lr)
    learn.fit_one_cycle(10, lr)

    plt.figure()
    learn.recorder.plot_metrics()
    plt.title("Training accuracies - "+subset)
    plt.savefig("./graphics/"+subset+"_train_accs.png")
    plt.savefig("./graphics/"+subset+"_train_accs.pdf")
    plt.close()

    plt.figure()
    learn.recorder.plot_lr()
    plt.title("Training Learning Rates - "+subset)
    plt.savefig("./graphics/"+subset+"_train_lr.png")
    plt.savefig("./graphics/"+subset+"_train_lr.pdf")
    plt.close()

    plt.figure()
    learn.recorder.plot_losses()
    plt.title("Training Losses - "+subset)
    plt.savefig("./graphics/"+subset+"_train_losses.png")
    plt.savefig("./graphics/"+subset+"_train_losses.pdf")
    plt.close()

    learn.export(subset+".pkl")

def testing(subset):
    def plot_confusion_matrix(cm, classes,
                              normalize=False,
                              title='Confusion matrix',
                              cmap=plt.cm.Blues):
        """
        This function prints and plots the confusion matrix.
        Normalization can be applied by setting `normalize=True`.

        Original source: scikit-learn documentation
        """
        if normalize:
            cm = cm.astype('float') / cm.sum(axis=1)[:, np.newaxis]
            title = title+" (normalized) \n"
        else:
            title = title+"\n"

        plt.imshow(cm, interpolation='nearest', cmap=cmap)
        plt.title(title)
        plt.colorbar()
        tick_marks = np.arange(len(classes))
        plt.xticks(tick_marks, classes)
        plt.yticks(tick_marks, classes)

        fmt = '.2f' if normalize else 'd'
        thresh = cm.max() / 2.
        for i, j in itertools.product(range(cm.shape[0]), range(cm.shape[1])):
            plt.text(j, i, format(cm[i, j], fmt),
                     horizontalalignment="center",
                     color="white" if cm[i, j] > thresh else "black")

        plt.ylabel('True label')
        plt.xlabel('Predicted label')
        plt.tight_layout()

    test = pd.read_csv("./data/"+subset+"_test.csv", usecols=["file", "label"])
    y_true = test["label"].values
    learn = load_learner("./", subset+".pkl",
                         test=ImageList.from_df(test, path=""))
    y_pred, _ = learn.get_preds(ds_type=DatasetType.Test)
    y_pred = np.argmax(y_pred.numpy(), axis=1)
    acc = skmetrics.accuracy_score(y_true, y_pred)
    title = subset+" Acc:"+str(np.round(acc, 4))+" confusion matrix"
    classes = sorted(np.unique(y_true))

    plt.figure(figsize=(18, 8))
    plt.subplot(1, 2, 1)
    plot_confusion_matrix(skmetrics.confusion_matrix(
        y_true=y_true, y_pred=y_pred), classes, title=title)
    plt.subplot(1, 2, 2)
    plot_confusion_matrix(skmetrics.confusion_matrix(
        y_true=y_true, y_pred=y_pred), classes, title=title, normalize=True)
    plt.savefig("./graphics/"+subset+"_confusion_matrix.png")
    plt.savefig("./graphics/"+subset+"_confusion_matrix.pdf")
    plt.close()

def main():
    tick = time.time()
    print("fastai version:", fastai.__version__)
    print("PyTorch version:", torch.__version__)

    subsets = ["dacon/comp6/"]

    for subset in subsets:
        tick1 = time.time()
        print("Running fastai on", subset)
        training(subset)
        gc.collect()
        testing(subset)
        print("Subset finished after:", time.time()-tick, "seconds")
        gc.collect()
    print("Finished after:", (time.time()-tick)/3600, "hours")

if __name__ == "__main__":
    main()


import unittest
from pygamehelper import *
from pygame import *
from vec2d import vec2d
from animation import Animation
from drone import Drone
from resource import Ore
from buildings import Building, MainBase, UpgradeFactory, \
                    Generator, Whearhouse, Outpost, Medbay

class TestMain():
    def __init__(self):
        main = MainBase()
        main.pos = vec2d(300, 300)
        main.target = vec2d(main.pos[0] + 10, main.pos[1])
        out = Outpost()
        out.pos = vec2d(200, 200)
        out.target = vec2d(out.pos[0] + 10, out.pos[1])
        self.buildings = [main, out]

class DronesTest(unittest.TestCase):

    def test_init_main_base(self):
        firstbuild = MainBase()
        firstbuild.pos = vec2d(300, 300)
        firstbuild.target = vec2d(firstbuild.pos[0] + 10, firstbuild.pos[1])
        self.assertEqual(firstbuild.pos[0], 300)
        self.assertEqual(firstbuild.pos[1], 300)

    def test_vec2d(self):
        testvec = vec2d(200,300)
        self.assertEqual(testvec[0], 200)
        self.assertEqual(testvec[1], 300)

    def test_mine(self):
        p = TestMain()
        drone = Drone()
        drone.pos = vec2d(100, 100)
        target = vec2d(100, 100)
        drone.target.append(target)
        ore = Ore()
        ore.pos = vec2d(100, 100)
        ore.mine(p, drone)
        self.assertEqual(drone.inventory[1][1], 1)
        self.assertEqual(ore.quantity, 799999)

    def test_drone_update(self):
        drone = Drone()
        drone.pos = vec2d(100, 100)
        target = vec2d(200, 200)
        drone.target.append(target)
        drone.update()
        self.assertEqual(drone.pos, (103, 103))
        self.assertEqual(drone.power, 255-0.1)

    def test_mainbuilding_buy_drone(self):
        p = TestMain()
        example = Drone()
        created = p.buildings[0].buy_drone()
        self.assertEqual(p.buildings[0].inventory[1][1], 490000)
        self.assertEqual(type(created), type(example))

    def test_mainbuilding_buy_battery(self):
        p = TestMain()
        p.buildings[0].buy_battery()
        self.assertEqual(p.buildings[0].inventory[1][1], 495000)
        self.assertEqual(p.buildings[0].inventory[0][1], 3)

    def test_mainbuilding_update(self):
        p = TestMain()
        p.buildings[0].update()
        self.assertEqual(p.buildings[0].power, 100005)

    def test_outpost_upgrade(self):
        p = TestMain()
        self.assertEqual(p.buildings[1].level, 0)
        p.buildings[1].upgrade()
        self.assertEqual(p.buildings[1].level, 1)
        self.assertEqual(p.buildings[1].fire_target, p.buildings[1].pos)
        self.assertEqual(p.buildings[1].target, p.buildings[1].pos)

if __name__ == '__main__':
    unittest.main()


from __future__ import annotations

import datetime
import logging
from dataclasses import dataclass
from typing import Optional

import pandas as pd
from dbt_semantic_interfaces.dataclass_serialization import SerializableDataclass
from dbt_semantic_interfaces.type_enums.time_granularity import TimeGranularity

from metricflow.time.time_granularity import offset_period

logger = logging.getLogger(__name__)

@dataclass(frozen=True)
class TimeRangeConstraint(SerializableDataclass):
    """Describes how the time dimension for metrics should be constrained."""

    start_time: datetime.datetime
    end_time: datetime.datetime

    def __post_init__(self) -> None:  # noqa: D
        if self.start_time > self.end_time:
            logger.warning(f"start_time must not be > end_time. start_time={self.start_time} end_time={self.end_time}")

        if self.start_time < TimeRangeConstraint.ALL_TIME_BEGIN():
            logger.warning(f"start_time={self.start_time} exceeds the limits of {TimeRangeConstraint.ALL_TIME_BEGIN()}")

        if self.end_time > TimeRangeConstraint.ALL_TIME_END():
            raise RuntimeError(f"end_time={self.end_time} exceeds the limits of {TimeRangeConstraint.ALL_TIME_END()}")

    @staticmethod
    def ALL_TIME_BEGIN() -> datetime.datetime:  # noqa: D
        return datetime.datetime(2000, 1, 1)

    @staticmethod
    def ALL_TIME_END() -> datetime.datetime:  # noqa: D
        return datetime.datetime(2040, 12, 31)

    @staticmethod
    def all_time() -> TimeRangeConstraint:
        """Return the range representing all time.

        This could also be represented with None as the ends, but doing this makes the logic simpler in many cases.
        """
        return TimeRangeConstraint(
            start_time=TimeRangeConstraint.ALL_TIME_BEGIN(),
            end_time=TimeRangeConstraint.ALL_TIME_END(),
        )

    @staticmethod
    def empty_time() -> TimeRangeConstraint:
        """Return the range representing no time."""
        return TimeRangeConstraint(
            start_time=TimeRangeConstraint.ALL_TIME_BEGIN(),
            end_time=TimeRangeConstraint.ALL_TIME_BEGIN(),
        )

    def _adjust_time_constraint_start_by_window(
        self,
        time_granularity: TimeGranularity,
        time_unit_count: int,
    ) -> TimeRangeConstraint:
        """Moves the start of the time constraint back by 1 window.

        if the metric is weekly-active-users (ie window = 1 week) it moves time_constraint.start one week earlier
        """
        start_ts = pd.Timestamp(self.start_time)
        offset = offset_period(time_granularity) * time_unit_count
        adjusted_start = (start_ts - offset).to_pydatetime()
        return TimeRangeConstraint(
            start_time=adjusted_start,
            end_time=self.end_time,
        )

    def adjust_time_constraint_for_cumulative_metric(
        self, granularity: Optional[TimeGranularity], count: int
    ) -> TimeRangeConstraint:
        """Given a time constraint for the overall query, adjust it to cover the time range for this metric."""
        if granularity is not None:
            return self._adjust_time_constraint_start_by_window(granularity, count - 1)

        # if no window is specified we want to accumulate from the beginning of time
        return TimeRangeConstraint(
            start_time=TimeRangeConstraint.ALL_TIME_BEGIN(),
            end_time=self.end_time,
        )

    def is_subset_of(self, other: TimeRangeConstraint) -> bool:  # noqa: D
        return self.start_time >= other.start_time and self.end_time <= other.end_time

    def __str__(self) -> str:  # noqa: D
        return f"[{self.start_time.isoformat()}, {self.end_time.isoformat()}]"

    def __repr__(self) -> str:  # noqa: D
        return (
            f"{self.__class__.__name__}(start_time='{self.start_time.isoformat()}', "
            f"end_time='{self.end_time.isoformat()}')"
        )

    def intersection(self, other: TimeRangeConstraint) -> TimeRangeConstraint:  # noqa: D
        # self is completely before the other
        if self.end_time < other.start_time:
            return TimeRangeConstraint.empty_time()
        # self starts before the start of other, and self ends within other
        elif self.start_time <= other.start_time <= self.end_time <= other.end_time:
            return TimeRangeConstraint(
                start_time=other.start_time,
                end_time=self.end_time,
            )
        # self starts before the start of other, and self ends after other
        elif self.start_time <= other.start_time <= other.end_time <= self.end_time:
            return other
        # self starts after the start of other, and self ends within other:
        elif other.start_time <= self.start_time <= self.end_time <= other.end_time:
            return self
        # self starts after the start of other, and self ends after other:
        elif other.start_time <= self.start_time <= other.end_time <= self.end_time:
            return TimeRangeConstraint(
                start_time=self.start_time,
                end_time=other.end_time,
            )
        # self is completely after other
        elif self.start_time > other.end_time:
            return TimeRangeConstraint.empty_time()
        else:
            raise RuntimeError(f"Unhandled case - self: {self} other: {other}")


import os
import sys
from pathlib import Path
import logging

# Path: template.py
logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(name)s - %(levelname)s - %(message)s')

project_name = "CNN_classification" # project name

list_of_files = [".github/workflows/.gitkeep",
                f"src/{project_name}/__init__.py",
                f"src/{project_name}/components/__init__.py",
                f"src/{project_name}/utils/__init__.py",
                f"src/{project_name}/config/configuration.py",
                f"src/{project_name}/config/__init__.py",
                f"src/{project_name}/pipeline/__init__.py",
                f"src/{project_name}/entity/__init__.py",
                f"src/{project_name}/constants/__init__.py",
                "config/config.yaml",
                "dvc.yaml",
                "requirements.txt",
                "setup.py",
                "research/trials.ipynb",
                "templates/index.html",

                ]

for file in list_of_files:
    path = Path(file)
    filedir , filename = os.path.split(path)
    if filedir !="" :
        os.makedirs(filedir,exist_ok=True)
        logging.info(f"Created {filedir}")
    if not os.path.exists(path) or os.path.getsize(path) == 0:
        with open(path,"w") as f:
            pass
        logging.info(f"Created {path}")
    else:
        logging.info(f"{path} already exists")