import geopandas as gp
import shapely

def intersection(left, right, grid_size=0):
    """Intersect the geometries from the left with the right.

    New, intersected geometries are stored in "geometry_right".

    Uses spatial index operations for faster operations.  Wholly contained
    geometries from right are copied intact, only those that intersect but are
    not wholly contained are intersected.

    Parameters
    ----------
    left : GeoDataFrame
    right : GeoDataFrame
    grid_size : int, optional (default: None)
        if present, geometries and results from intersection will be
        snapped to this precision grid

    Returns
    -------
    DataFrame or None
        output geometries are in "geometry_right"
        None if there are no intersections
    """

    tree = shapely.STRtree(right.geometry.values)
    ix = tree.query(left.geometry.values, predicate="intersects")

    if len(ix[0]) == 0:
        return None

    # copy original geometries; they will be clipped below if needed
    intersects = gp.GeoDataFrame(
        {
            "geometry": left.geometry.values.take(ix[0]),
            "index_right": right.index.values.take(ix[1]),
            "geometry_right": right.geometry.values.take(ix[1]),
        },
        index=left.index.take(ix[0]),
        crs=left.crs,
    )

    shapely.prepare(intersects.geometry.values)
    contains = shapely.contains_properly(
        intersects.geometry.values, intersects.geometry_right.values
    )

    # clip any that are not fully contained
    tmp = intersects[~contains]
    intersects.loc[~contains, "geometry_right"] = shapely.intersection(
        tmp.geometry.values, tmp.geometry_right.values, grid_size=grid_size
    )

    return left.join(intersects.drop(columns=["geometry"]), how="inner").join(
        right.drop(columns=["geometry"]), on="index_right"
    )


import os
from flask import jsonify, request
import requests

def getWeather():
    try:
        API_KEY = os.getenv("API_KEY")
        args = request.args
        lon = str(args.get('lon'))
        lat = str(args.get('lat'))

        url = f'https://api.openweathermap.org/data/2.5/weather?lat={lat}&lon={lon}&appid={API_KEY}'
        response = requests.get(url).json()
        return jsonify(response),200

    except Exception as e:
      raise Exception("Something bad happened")


##########################################
# @subject : Person segmentation         #
# @author  : perryxin                    #
# @date    : 2018.12.27                  #
##########################################
import torch.utils.data as Data
from read_data import *
from config import *
from models.unet_plusplus import *
from models.linknet import *

model_name = 'linknet'  # 'unet++'
if model_name == 'unet++':
    net = Unet_2D(3, 1, 'test')
    checkpoint = torch.load("../models/weights/unet++_11.pth", map_location='cpu')
    dataset_test = MyData(istrain="test", size=256)
else:  # 'linknet'
    net = LinkNet()
    checkpoint = torch.load("../models/weights/link_640_57.pth", map_location='cpu')
    dataset_test = MyData(istrain="test", size=640)
net.load_state_dict(checkpoint['net'])
loader_test = Data.DataLoader(dataset_test, batch_size=conf.BATCH_SIZE_TEST, shuffle=False)

net.cuda()
print("test_images", len(dataset_test))
print("start testing...")

# val###################
net.eval()
test_iou = 0
t1 = time.time()
for i, (img, label) in enumerate(loader_test):

    output = net(img.float()).cuda()
    output[output >= 0.5] = 1.
    output[output != 1] = 0.
    iou_ = iou(output.cpu(), label)
    test_iou += iou_
    print("img_%d: iou=%.4f" % (i, iou_))
    #############show
    isShow = True
    if isShow:
        import matplotlib.pyplot as plt

        plt.subplot(221)
        output = output[0, 0].detach().numpy()  # .detach().cpu()
        label = label[0, 0].numpy()
        img = (img[0].permute(1, 2, 0).detach().numpy() * conf.std + conf.mean)  # .detach().cpu()
        img = img * 255
        plt.imshow(np.uint8(img))
        plt.title("origin")
        plt.axis('off')
        plt.xticks([])
        plt.yticks([])
        plt.subplot(222)
        plt.imshow(label)
        plt.title("label")
        plt.axis('off')
        plt.xticks([])
        plt.yticks([])
        plt.subplot(223)
        mm = apply_mask(img, output, color=random_colors(1)[0])
        # mm=cv2.addWeighted(np.uint8(img),0.8,np.uint8(output*255),0.2,0)
        plt.imshow(np.uint8(mm))
        plt.title("origin+seg")
        plt.axis('off')
        plt.xticks([])
        plt.yticks([])
        plt.subplot(224)
        plt.imshow(output)
        plt.title("seg")

        plt.axis('off')
        plt.xticks([])
        plt.yticks([])
        # plt.savefig("./results/imgs/img_%d.png"%i)
        plt.show()

    del img, label
test_iou /= len(loader_test)
t2 = time.time()
print("speed: %.4f fps, test_iou : %.4f" % (len(dataset_test) / (t2 - t1), test_iou))


import cv2
import io
import os

from google.cloud import vision

os.environ["GOOGLE_APPLICATION_CREDENTIALS"] = ".." #put your google API credintials here

# Instantiates a client
client = vision.ImageAnnotatorClient()

#folder = "/uploads/"
class Image():
    def __init__(self):
        self.labels = []
        self.image = None

    def capture(self):
        # chi 7aja
        cam = cv2.VideoCapture(0)
        cv2.namedWindow("Camera")
        img_counter = 0
        while True:
            ret, frame = cam.read()
            if not ret:
                print("failed to grab frame")
                break
            cv2.imshow("test", frame)

            k = cv2.waitKey(1)
            if k % 256 == 27:
                # ESC pressed
                print("Escape hit, closing...")
                break
            elif k % 256 == 32:
                # SPACE pressed
                img_name = "opencv_frame_{}.png".format(img_counter)
                cv2.imwrite(img_name, frame)
                print("{} written!".format(img_name))
                img_counter += 1
        cam.release()
        cv2.destroyAllWindows()
        self.image = img_name

    def capture_bis(self):
        # chi 7aja

        # initialize the camera
        cam = cv2.VideoCapture(0)  # 0 -> index of camera
        s, img = cam.read()
        img_counter = 0
        if s:  # frame captured without any errors
            #namedWindow("cam-test", CV_WINDOW_AUTOSIZE)
            #imshow("cam-test", img)
            cv2.waitKey(0)
            #destroyWindow("cam-test")
            img_name = "opencv_frame_{}.png".format(img_counter)
            cv2.imwrite(img_name, img)  # save image
        self.image = img_name

    def findLabels(self):
        # The name of the image file to annotate
        file_name = self.image

        # Loads the image into memory
        with io.open(file_name, 'rb') as image_file:
            content = image_file.read()

        image = vision.Image(content=content)

        # Performs label detection on the image file
        response = client.label_detection(image=image)
        labels = response.label_annotations

        #self.labels = labels
        print('Labels:')
        for label in labels:
            print(label.description+', score: '+str(label.score))
            self.labels.append(label)


import gensim
from sklearn import svm
from sklearn import datasets
from sklearn.feature_extraction.text import TfidfVectorizer
from joblib import dump, load
import flask
from flask_restful import Resource, Api, reqparse
from flask import request
from functools import partial
from flask import request, jsonify

app = flask.Flask(__name__)
app.config["DEBUG"] = True

def k_means_sort(e, product_terms, query):
	#global product_terms, query
	occurences = 0
	for word in [x.lower() for x in e.split(',')[1].split('-')]:
		if word in product_terms or word in query.split(' '):
			occurences += 1
	occurences /= len(e.split(',')[1].split('-'))
	average_similarity = 0
	counter = 0
	for word in [x.lower() for x in e.split(',')[1].split('-')]:
		for query_word in query.split(' '):
			if query_word in word2vec_model.wv.vocab and word in word2vec_model.wv.vocab:
				average_similarity += word2vec_model.similarity(query_word, word)
				counter += 1
	average_similarity /= (counter if counter != 0 else 1)
	return occurences + average_similarity

word2vec_model = gensim.models.Word2Vec.load('../MachineLearningModels/Word2Vec/word2vec.model')
vectorizing_model = load('../MachineLearningModels/KMeansClustering/vectorizing_model.sav')
clustering_model = load('../MachineLearningModels/KMeansClustering/clustering_model.sav')
products = open('../formatted_data/all_products.txt').read().split('\n')[:-1]

@app.route('/', methods=['GET'])
def home():
	parser = request.args
	#parser.add_argument('query', type=str)
	query = parser['query']
	prediction = clustering_model.predict(vectorizing_model.transform([query]))[0]
	product_terms = [vectorizing_model.get_feature_names()[product_term_index] for product_term_index in clustering_model.cluster_centers_.argsort()[:, ::-1][prediction, :30]]
	relavant_products = products.copy()
	relavant_products.sort(key=partial(k_means_sort, product_terms=product_terms, query=query), reverse=True)
	return jsonify(relavant_products[:10])
	#return "Distant Reading Archive: This site is a prototype API for distant reading of science fiction novels."

app.run()


"""
This module is used to store the methods for setting up the NSX-T XUI
"""
import json
import os
from os import path
from packaging import version

from cbhooks.models import CloudBoltHook
from servicecatalog.models import ServiceBlueprint
from resourcehandlers.models import ResourceHandler
from xui.nsxt.xui_utilities import check_for_nsxt, setup_nsx_tags
from django.utils.text import slugify
from utilities.logger import ThreadLogger

logger = ThreadLogger(__name__)

XUI_PATH = path.dirname(path.abspath(__file__))
XUI_NAME = XUI_PATH.split("/")[-1]
CONFIG_FILE = f'/var/opt/cloudbolt/proserv/xui/xui_versions.json'

def get_data_from_config_file(property_key):
    with open(CONFIG_FILE, 'r') as f:
        config = json.load(f)
        data = config[XUI_NAME][property_key]
    return data

# If we find a Blueprint with the same name, should it be overwritten?
try:
    OVERWRITE_EXISTING_BLUEPRINTS = get_data_from_config_file(
        'OVERWRITE_EXISTING_BLUEPRINTS')
except Exception:
    OVERWRITE_EXISTING_BLUEPRINTS = True
# From what I can tell, when a Blueprint is using a remote source, the actions
# are only updated at initial creation. Setting this toggle to True would
# set each action to use the remote source - forcing update of the actions when
# the XUI gets updated
try:
    SET_ACTIONS_TO_REMOTE_SOURCE = get_data_from_config_file(
        'SET_ACTIONS_TO_REMOTE_SOURCE')
except Exception:
    SET_ACTIONS_TO_REMOTE_SOURCE = True

def run_config(xui_version):
    config_needed = False
    try:
        with open(CONFIG_FILE, 'r') as f:
            config = json.load(f)
            current_version = config[XUI_NAME]["current_version"]
            if version.parse(current_version) < version.parse(xui_version):
                logger.info(f"Current Version: {current_version} is less than"
                            f" {xui_version}. Running config.")
                config_needed = True
    except Exception:
        logger.info(f"Config file not found going to run configuration")
        config_needed = True
    if config_needed:
        logger.info("Running Configuration")
        configure_xui()
        try:
            config
        except NameError:
            config = {}
        config[XUI_NAME] = {
            "current_version": xui_version,
            "SET_ACTIONS_TO_REMOTE_SOURCE": SET_ACTIONS_TO_REMOTE_SOURCE,
            "OVERWRITE_EXISTING_BLUEPRINTS": OVERWRITE_EXISTING_BLUEPRINTS
        }
        with open(CONFIG_FILE, 'w') as f:
            json.dump(config, f, indent=4)

def configure_xui():
    configure_tags()
    configure_blueprints()

def configure_blueprints():
    blueprints_dir = f'{XUI_PATH}/blueprints/'
    for bp in os.listdir(blueprints_dir):
        bp_dir = f'{blueprints_dir}{bp}/'
        bp_path = f'{bp_dir}{bp}.json'
        with open(bp_path, 'r') as f:
            bp_json = json.load(f)
        bp_name = bp_json["name"]
        try:
            bp_global_id = bp_json["id"]
        except KeyError:
            logger.warning(f"Blueprint: {bp_name} does not have an id. "
                           f"Skipping")
            continue

        bp, created = ServiceBlueprint.objects.get_or_create(
            global_id=bp_global_id,
            status='ACTIVE'
        )
        if not created:
            if OVERWRITE_EXISTING_BLUEPRINTS:
                logger.info(f"Overwriting Blueprint: {bp_name}")
            else:
                logger.info(f"Blueprint: {bp_name} already exists. Skipping")
                continue
        bp.remote_source_url = f'file://{bp_path}'
        bp.save()
        bp.refresh_from_remote_source()
        logger.info(f"Finished refreshing: {bp_name} from remote source")
        set_actions_to_remote_source(bp_dir, bp_json, created)

def set_actions_to_remote_source(bp_dir, bp_json, created):
    if SET_ACTIONS_TO_REMOTE_SOURCE or created:
        logger.info(f'Starting to set actions to remote source for BP: '
                    f'{bp_json["name"]}')
        action_datas = []  # Tuples of (action_name, action_path)
        elements = ["teardown_items", "deployment_items", "management_actions"]
        for element in elements:
            for action in bp_json[element]:
                action_data = get_action_data(action, bp_dir, element)
                action_datas.append(action_data)
        for action_data in action_datas:
            action_name, action_path = action_data
            logger.info(f"Setting action: {action_name} to remote source")
            set_action_to_remote_source(action_name, action_path)
    else:
        logger.info("Not setting actions to remote source. Update the "
                    "SET_ACTIONS_TO_REMOTE_SOURCE variable to True if you "
                    "want to do this")
    return None

def set_action_to_remote_source(action_name, action_path):
    try:
        action = CloudBoltHook.objects.get(name=action_name)
        action.source_code_url = f'file://{action_path}'
        action.save()
    except:
        logger.warning(f"Could not find action: {action_name}, will not be "
                       f"able to set to remote source")

def get_action_data(action, bp_dir, item_name):
    if item_name == 'management_actions':
        file_name = slugify(action["label"]).replace("-", "_")
        json_file = f'{file_name}.json'
        json_path = f'{bp_dir}{file_name}/{file_name}/{json_file}'
        action_name = action["label"]
    else:
        file_name = slugify(action["name"]).replace("-", "_")
        json_file = f'{file_name}.json'
        json_path = f'{bp_dir}{file_name}/{file_name}.json'
        action_name = action["name"]
    action_path = get_action_path_from_json(json_path, json_file)
    return action_name, action_path

def get_action_path_from_json(json_path, json_file):
    with open(json_path, 'r') as f:
        action_json = json.load(f)
    action_file = action_json["script_filename"]
    action_path = json_path.replace(json_file, action_file)
    return action_path

def configure_tags():
    # Create the nsxt_tag parameter if it does not already exist
    cf, _ = setup_nsx_tags()

    # Add an NSXT parameter to any NSXT environments
    rhs = ResourceHandler.objects.all()
    for rh in rhs:
        if check_for_nsxt(rh):
            for env in rh.environment_set.all():
                env.custom_fields.add(cf)


import pdb, argparse
import numpy as np
import matplotlib.pyplot as plt

import torch
import torch.nn as nn
import torch.optim as optim
import torchvision as thv
from torchvision import transforms

from torch.utils.data import Dataset, DataLoader
from CarlaDataset import CarlaDataset
from CVAE import CVAE
from siameseCVAE import siameseCVAE

def main():
    model = siameseCVAE()
    criterion = nn.MSELoss()
    optimizer = optim.Adam(model.parameters(), lr=0.001, betas=(0.9, 0.999),
                            eps=1e-08, weight_decay=0)
    epochs = 10
    dl = DataLoader(CarlaDataset('../data/'))
    for epoch in range(epochs):
        for i, X in enumerate(dl):
            img = X[0]
            img_ = X[1]
            ctrl_inputs = X[2]

            img = (img/255).float()
            img_ = (img_/255).float()
            img1 = img[:,0,:,:,:]
            img2 = img[:,1,:,:,:]
            img1_ = img_[:,0,:,:,:]
            img2_ = img_[:,1,:,:,:]

            xhat, yhat, z, z_mean, z_stdev = model.forward(img1,img2,ctrl_inputs)
            lossx = criterion(xhat,img1_)
            lossy = criterion(yhat,img2_)
            optimizer.zero_grad()
            lossx.backward()
            lossy.backward()

            if (i+1) % 10 == 0:
              print ('Epoch [{}/{}], Step [{}/{}], Loss: {:.4f}'.format(epoch+1, epochs, i+1, total_step, loss.item()))

            pdb.set_trace()

if __name__ == '__main__':
    parser = argparse.ArgumentParser(description='PyTorch MNIST Example')
    parser.add_argument('--batch_size', type=int, default=32)
    # parser.add_argument('--lr', type=int, default=0.005)

    args = parser.parse_args()

    main()


#!/bin/python

import argparse
import numpy as np
import pandas as pd
import time
import json
from datetime import datetime, timedelta
from confluent_kafka.avro import AvroProducer

def parse_args():
    parser = argparse.ArgumentParser(description='Publish CSV records to a Kafka topic.')
    parser.add_argument('--delta-minutes', '-d', default=2, type=int, help='Minutes to shift timestamps')
    parser.add_argument('--input-file', '-i', default='../resources/ml-latest-small/ratings.csv',
                        help='Path to the CSV input file')
    parser.add_argument('--output-topic', '-o', default='user-ratings-4', help='Output Kafka topic')
    return parser.parse_args()

def transform_timestamps(df, start_time_adjusted, end_time_adjusted):
    start_time_adjusted_ts = int(start_time_adjusted.timestamp() * 1000)
    end_time_adjusted_ts = int(end_time_adjusted.timestamp() * 1000)

    # Generate random timestamps within the time range
    random_timestamps = np.random.randint(start_time_adjusted_ts, end_time_adjusted_ts, df.shape[0])

    # Sort the random timestamps
    sorted_timestamps = np.sort(random_timestamps)

    # Assign sorted random timestamps to the DataFrame
    df['timestamp'] = sorted_timestamps

    return df

def setup_producer():
    # Avro schema
    # key schema not used due to streams dependency confusing key and value schemas
    key_schema_str = json.dumps({
        "namespace": "ratings",
        "type": "record",
        "name": "UserID",
        "fields": [
            {"name": "userId", "type": "int"}
        ]
    })
    value_schema_str = json.dumps({
        "namespace": "ratings",
        "type": "record",
        "name": "Rating",
        "fields": [
            {"name": "userId", "type": "int"},
            {"name": "movieId", "type": "int"},
            {"name": "rating", "type": "float"},
            {
                "name": "timestamp",
                "type": {
                    "type": "long",
                    "logicalType": "timestamp-millis"
                }
            }
        ]
    })

    # Producer configuration
    producer = AvroProducer({
        'bootstrap.servers': 'localhost:9092',
        'schema.registry.url': 'http://localhost:8081'
    }, default_value_schema=value_schema_str)
    # }, default_value_schema=value_schema_str, default_key_schema=key_schema_str)

    return producer

def send_batch(producer, batch, output_topic):
    for value in batch:
        producer.produce(
            topic=output_topic,
            # key=dict(userId=value['userId']),
            value=value
        )

def manage_loop(df, producer, output_topic):
    total_count = df.shape[0]
    record_count = 0
    while not df.empty:
        current_utc_ts = int(datetime.utcnow().timestamp() * 1000)  # Convert this to UNIX milliseconds as well
        past_rows = df[df['timestamp'] <= current_utc_ts]

        if not past_rows.empty:
            batch = past_rows.to_dict('records')
            send_batch(producer, batch, output_topic)

            batch_count = len(batch)
            record_count += batch_count

            current_time_str = datetime.utcnow().strftime("%Y-%m-%d %H:%M:%S")
            print(f"{current_time_str} - {batch_count} rows sent ({record_count}/{total_count})")

            df = df[df['timestamp'] > current_utc_ts]
            producer.flush()

        time.sleep(1)

def main():
    args = parse_args()

    df = pd.read_csv(args.input_file)
    start_time_adjusted = datetime.utcnow()
    end_time_adjusted = start_time_adjusted + timedelta(minutes=args.delta_minutes)

    df = transform_timestamps(df, start_time_adjusted, end_time_adjusted)
    producer = setup_producer()

    print(f"Started publishing records at {start_time_adjusted}")
    manage_loop(df, producer, args.output_topic)
    print(f"Finished publishing records at {datetime.utcnow()}")

if __name__ == "__main__":
    main()


import random
import sys
import threading
import time
from multiprocessing import Process

import zmq

import requests
from Crypto.Hash import SHA3_256
from Crypto.PublicKey import ECC
from Crypto.Signature import DSS
from Pyro4.util import json

n = int(sys.argv[2])
t = int(sys.argv[4])
endpoint = "http://localhost:5000/peers"

def peer_func(id):
    mutex = threading.Lock()
    ID = id
    MyPORT = random.randrange(1300, 60000)
    sign_key = ECC.generate(curve='NIST P-256')
    verify_key = sign_key.public_key()
    peer = {
        "id": ID,
        "port": MyPORT,
        "public_key": verify_key.export_key(format='OpenSSH'),
        "random_numbers_list": []}
    res = requests.post(endpoint, json=peer)

    peer["list"] = requests.get(endpoint).json()
    peer["random_number"] = random.getrandbits(256)
    ports = []
    messages = []
    for i in range(n):
        ports.append(peer["list"][i]["port"])

    context = zmq.Context()
    num_sender = context.socket(zmq.PUB)
    num_sender.bind("tcp://127.0.0.1:" + str(MyPORT))

    time.sleep(1)

    contexts = []
    for k in range(n):
        context2 = zmq.Context()
        num_receiver = context2.socket(zmq.SUB)
        num_receiver.connect("tcp://127.0.0.1:" + str(ports[(id + k) % n]))
        time.sleep(0.5)
        num_receiver.subscribe("")
        contexts.append(num_receiver)

    time.sleep(1)

    num_sender.send_string(str(peer["random_number"]))

    for k in range(n):
        res = int(contexts[k].recv_string())
        messages.append(res)

    selection = 0

    for message in messages:
        selection = selection ^ message
        time.sleep(0.01)

    d = SHA3_256.new(selection.to_bytes(32, byteorder='big'))
    for k in range(t - 1):
        d = SHA3_256.new(d.digest())

    selection = int.from_bytes(d.digest(), "big") % n
    file = ""
    for message in messages:
        file += str(message) + "\n"
    file += str(selection) + "\n"

    time.sleep(1)
    signer = DSS.new(sign_key, 'fips-186-3')
    h = SHA3_256.new(file.encode('utf-8'))
    signature = signer.sign(h)
    file += str(int.from_bytes(signature, "big")) + "\n"
    file += verify_key.export_key(format='OpenSSH')

    time.sleep(1)
    file_writer = open("sample_election_" + str(id) + ".log", "w")
    file_writer.write(file)
    file_writer.close()

if __name__ == "__main__":
    procs = []
    for i in range(n):
        proc = Process(target=peer_func, args=(i,))
        procs.append(proc)
        proc.start()

    for proc in procs:
        proc.join()


from esphome.components import number
from esphome.const import CONF_MAX_VALUE, CONF_MIN_VALUE, CONF_STEP
import esphome.config_validation as cv
import esphome.codegen as cg

from .. import fourheat_config_validation as fhcv
from .. import (
    fourheat_ns,
    CONF_DATAPOINT,
    CONF_FOURHEAT_ID,
    CONF_PARSER,
    CONF_QUERY_DATAPOINT,
    FourHeat,
    get_parser_expression,
)

DEPENDENCIES = ["fourheat"]

FourHeatNumber = fourheat_ns.class_("FourHeatNumber", number.Number, cg.Component)

def validate_min_max(config):
    if config[CONF_MAX_VALUE] <= config[CONF_MIN_VALUE]:
        raise cv.Invalid("max_value must be greater than min_value")
    return config

CONFIG_SCHEMA = cv.All(
    number.number_schema(FourHeatNumber)
    .extend(
        {
            cv.GenerateID(CONF_FOURHEAT_ID): cv.use_id(FourHeat),
            cv.Required(CONF_DATAPOINT): fhcv.datapoint,
            cv.Optional(CONF_QUERY_DATAPOINT): fhcv.datapoint,

            cv.Optional(CONF_PARSER): cv.returning_lambda,

            cv.Optional(CONF_MAX_VALUE, default=16777215.0): cv.float_,
            cv.Optional(CONF_MIN_VALUE, default=-16777215.0): cv.float_,
            cv.Optional(CONF_STEP, default=1): cv.positive_float,
        }
    )
    .extend(cv.COMPONENT_SCHEMA),
    validate_min_max,
)

async def to_code(config):
    var = await number.new_number(
        config,
        min_value=config[CONF_MIN_VALUE],
        max_value=config[CONF_MAX_VALUE],
        step=config[CONF_STEP]
    )
    await cg.register_component(var, config)

    parent = await cg.get_variable(config[CONF_FOURHEAT_ID])
    cg.add(var.set_fourheat_parent(parent))

    cg.add(var.set_datapoint_id(config[CONF_DATAPOINT]))

    if CONF_QUERY_DATAPOINT in config:
        cg.add(var.set_query_datapoint_id(config[CONF_QUERY_DATAPOINT]))

    if CONF_PARSER in config:
        parser = await get_parser_expression(config[CONF_PARSER], int)
        cg.add(var.set_parser(parser))


from ase import Atoms
import numpy as np

atoms = Atoms(['O', 'H', 'H'], positions=[[0., 0., 0.119262],
                                          [0., 0.763239, -0.477047],
                                          [0., -0.763239, -0.477047]])

# Angle no pbc
assert abs(atoms.get_angle(1, 0, 2) - 104) < 1e-3

atoms.set_cell([2, 2, 2])

# Across different pbcs
atoms.set_pbc([True, False, True])
atoms.wrap()
assert abs(atoms.get_angle(1, 0, 2, mic=True) - 104) < 1e-3

# Across all True pbc
atoms.set_pbc(True)
atoms.wrap()
assert abs(atoms.get_angle(1, 0, 2, mic=True) - 104) < 1e-3

# Simple tetrahedron
tetra_pos = np.array([[0, 0, 0], [1, 0, 0], [.5, np.sqrt(3) * .5, 0],
                      [.5, np.sqrt(1/3.) * .5, np.sqrt(2/3.)]])
atoms = Atoms(['H', 'H', 'H', 'H'],
              positions=tetra_pos - np.array([.2, 0, 0]))
angle = 70.5287793655
assert abs(atoms.get_dihedral(0, 1, 2, 3) - angle) < 1e-3

atoms.set_cell([3, 3, 3])
atoms.set_pbc(True)
atoms.wrap()
assert abs(atoms.get_dihedral(0, 1, 2, 3, mic=True) - angle) < 1e-3


from argparse import ArgumentParser, HelpFormatter
import itertools
import sys
from pathlib import Path

# ============================================================================================================
# === DTN ARGUMENT PARSER CLASSES
# ============================================================================================================

class DtnArgumentParser(ArgumentParser):
    ''' Subclass of Python's native ArgumentParser that incorporates a function to display warning messages related
        to the arguments.
    '''

    def __init__(self, *args, **kwargs):
        ''' Constructor simply calls ArgumentParser constructor '''
        super().__init__(*args, **kwargs)

    def warning(self, message):
        ''' Display a warning message related to one of the arguments. The format is consistent with that of errors

            :param str message: Warning message to display
        '''
        print('{}: WARNING: {}\n'.format(self.prog, message))

    def error(self, message):
        print('{}: ERROR: {}\n'.format(self.prog, message))
        self.print_help()
        sys.exit(-1)

# ============================================================================================================
# === DTN ARGUMENT PARSE FUNCTIONS
# ============================================================================================================

def get_argument_parser():
    ''' Parse the command-line arguments for this program

        :return ArchNetArgumentParser: The argument parser
    '''
    formatter = lambda prog: HelpFormatter(prog, max_help_position=50, width=200)
    parser    = DtnArgumentParser(prog='Dtn Simulator', formatter_class=formatter, description='DTN Network Simulator')

    # Add required arguments for running in config file mode
    parser.add_argument('-cf', '--configfile', help='configuration file path',
                        type=str, default=None, nargs='?')

    # Add optional arguments
    parser.add_argument('-v', '--validate', help='run unit tests',
                        action='store_true')

    return parser

def dict_to_args_list(args):
    argnames, argvals = zip(*args.items())
    argnames = ['--' + name for name in argnames]
    return itertools.chain.from_iterable(zip(argnames, argvals))

def process_arguments(args=None):
    ''' Process the arguments of the application and run ArchNet in batch or GUI mode depending on what is specified

        :param None or dict:
    '''
    # Get argument parser
    parser = get_argument_parser()

    # If no arguments provided, use argument parser
    if not args:
        args = parser.parse_args()
    elif isinstance(args, dict):
        args = parser.parse_args(dict_to_args_list(args))
    elif args != None:
        raise RuntimeError('process_arguments: args can only be None or a dictionary')

    if not args.configfile: return args

    # Check the validity of the configuration file
    configfile = Path(args.configfile)
    if not configfile.exists():
        raise FileExistsError(f'Configuration file {configfile} does not exist')
    if configfile.suffix not in ['.yaml', '.yml']:
        raise ValueError(f'The configuration file {configfile} is not a YAML file')

    return args


""" Assignment 6 performance of Mango
"""
from ast import stmt
from pathlib import Path
import csv
import pathlib
from timeit import repeat, timeit as timer
import main
import users
import user_status
import pymongo
import pandas as pd
import random
import string
import time
import socialnetwork_model

letters = string.ascii_lowercase
# from pymongo import MongoClient
# from pymongo.errors import DuplicateKeyError

def add_random_users():

    for n in range(500):
        new_user_id = ''.join(random.choice(letters) for i in range(10))
        add_user(new_user_id,"email","name","lastname",user_collection)

def add_random_status():

    """
    search a 1000 records
   """

    cdw = pathlib.Path.cwd()
    # print(f"Reading {filename}\npath is {cdw}")
    status_header = ["STATUS_ID","USER_ID","STATUS_TEXT"]
    #global tatus_header
    try:
        filename = "status_updates.csv"
        with open(filename, mode='r') as statusfile:
            reader = csv.reader(statusfile, delimiter=',')
            status_header = next(reader)
            count = 0
            # print(status_header)
            for line in reader:

                new_status_id = ''.join(random.choice(letters) for i in range(10))
                add = main.add_status(new_status_id,line[1],line[2],status_collection)

                count += 1
                if count == 1000:
                    break
                # print("added:",line[0], line[1], line[2])
        statusfile.close()

        # print(f'lines counted for Status: {count}')
        return True

    except FileNotFoundError:
        print(f"ERROR: Couldn't find {filename}")
        print(f"Current path is: {cdw} ")

def load_users(filename, user_collection):
    """
    Load 1000 lines
    """

    user_header = ["USER_ID", "EMAIL", "NAME", "LASTNAME"]
    cdw = pathlib.Path.cwd()
    # print(f"Reading{cdw} {filename}\npath is {cdw}")
    try:
        myclient = pymongo.MongoClient("mongodb://localhost:27017/")
        mydb = myclient["UserAccounts"]
        # mongo_docs = mydb["Users"]
        usercol = mydb["UserAccounts"]
        # mongo_docs =
        # print("drop the database")
        # usercol.drop()

        # df =  pd.DataFrame(list(mongo_docs))
        # df.to_csv(filename, index=False)
    # return True

    except FileNotFoundError:

        print(f"ERROR: Couldn't find {filename}")
        print(f"Current path is: {cdw} ")
        path = Path(filename)
        print(path)
        return False

    except OSError:
        return False

    count = 0
    try:
        with open(filename, mode='r', newline='\n') as accountsfile:
            # print(accountsfile)
            reader = csv.reader(accountsfile, delimiter=',')
            user_header = next(reader)
            # print("header", user_header)

            for row in reader:
                count += 1
                # USER_ID,NAME,LASTNAME,EMAIL (self, user_id, email, user_name, user_last_name)
                # print("row",row[0], row[1], row[2], row[3])

                    # print("line",row)
                load = add_user(row[0], row[3], row[1], row[2], user_collection)
                if count == 1000:
                    break

            # print(f"lines counted {count}")

        accountsfile.close()
        return True
    except FileNotFoundError:
            print(f"ERROR: Couldn't find {filename}")
            print(f"Current path is: {cdw} ")
            return False

def search_1000_status_updates(filename, status_collection):
    """
    search a 1000 records
   """

    cdw = pathlib.Path.cwd()
    # print(f"Reading {filename}\npath is {cdw}")
    status_header = ["STATUS_ID","USER_ID","STATUS_TEXT"]
    #global tatus_header
    try:

        with open(filename, mode='r') as statusfile:
            reader = csv.reader(statusfile, delimiter=',')
            status_header = next(reader)
            count = 0
            # print(status_header)
            for line in reader:

                search = main.search_status(line[0], status_collection)

                count += 1
                if count == 1000:
                    break
                # print("added:",line[0], line[1], line[2])
        statusfile.close()

        # print(f'lines counted for Status: {count}')
        return True

    except FileNotFoundError:
        print(f"ERROR: Couldn't find {filename}")
        print(f"Current path is: {cdw} ")

        return False

def delete_status_updates(filename, status_collection):
    """
    search a 1000 records
   """

    cdw = pathlib.Path.cwd()
    # print(f"Reading {filename}\npath is {cdw}")
    status_header = ["STATUS_ID","USER_ID","STATUS_TEXT"]

    #global tatus_header
    try:

        with open(filename, mode='r') as statusfile:
            reader = csv.reader(statusfile, delimiter=',')
            status_header = next(reader)
            count = 0
            # print(status_header)
            for line in reader:

                search = main.delete_status(line[0], status_collection)

                count += 1
                if count == 1000:
                    break
                # print("added:",line[0], line[1], line[2])
        # tatusfile.close()

        # print(f'lines counted for Status: {count}')
        return True

    except FileNotFoundError:
        print(f"ERROR: Couldn't find {filename}")
        print(f"Current path is: {cdw} ")

        return False

def search_1000_users(filename, user_collection):
    """
    Load 1000 lines
    """
    user_header = ["USER_ID", "EMAIL", "NAME", "LASTNAME"]
    cdw = pathlib.Path.cwd()
    count = 0
    try:
        with open(filename, mode='r', newline='\n') as accountsfile:
            # print(accountsfile)
            reader = csv.reader(accountsfile, delimiter=',')
            user_header = next(reader)
            # print("header", user_header)

            for row in reader:
                count += 1
                # USER_ID,NAME,LASTNAME,EMAIL (self, user_id, email, user_name, user_last_name)
                # print("row",row[0], row[1], row[2], row[3])

                    # print("line",row)
                load = main.search_user(row[0], user_collection)

                if count == 1000:
                    break
        accountsfile.close()
            # print(f"lines counted {count}")
    except FileNotFoundError:
            print(f"ERROR: Couldn't find {filename}")
            print(f"Current path is: {cdw} ")
            return False

def delete_1000_users(filename, user_collection):
    """
    delete 1000 lines
    """
    user_header = ["USER_ID", "EMAIL", "NAME", "LASTNAME"]
    cdw = pathlib.Path.cwd()
    count = 0
    try:
        with open(filename, mode='r', newline='\n') as accountsfile:
            # print(accountsfile)
            reader = csv.reader(accountsfile, delimiter=',')
            user_header = next(reader)
            # print("header", user_header)

            for row in reader:
                count += 1
                # USER_ID,NAME,LASTNAME,EMAIL (self, user_id, email, user_name, user_last_name)
                # print("row",row[0], row[1], row[2], row[3])

                load = main.delete_user(row[0], user_collection)

                if count == 1000:
                    break
        accountsfile.close()
            # print(f"lines counted {count}")

        return True
    except FileNotFoundError:
            print(f"ERROR: Couldn't find {filename}")
            print(f"Current path is: {cdw} ")
            return False

def init_user_collection():
    '''
    Creates and returns a new instance of UserCollection
    '''
    return users.UserCollection()

def init_status_collection():
    """

    Creates and returns a new instance of UserStatusCollection
    UserStatusCollection()
    """

    return user_status.UserStatusCollection()

def add_user(user_id, email, user_name, user_last_name, user_collection):
    """add users with users: def add_user(self, user_id, email, user_name, user_last_name):
        Creates a new instance of User and stores it in user_collection
        (which is an instance of UserCollection)
    """
    result = user_collection.add_user(user_id, email, user_name, user_last_name)
    #print(user_id, email, user_name, user_last_name)
    if result:

        return True
    else:

        return False

def add_status(status_id, user_id, status_text, status_collection):
    """
    Creates a new instance of UserStatus and stores it in
    user_collection(which is an instance of UserStatusCollection)

    Requirements:
    - status_id cannot already exist in user_collection.
    - Returns False if there are any errors (for example, if
      user_collection.add_status() returns False).
    - Otherwise, it returns True.
    """
    status_collection = main.add_status(status_id, user_id, status_text,status_collection)
    return status_collection

def load_status_updates(filename, status_collection):
    """
    Asd a 1000 records
   """
    try:
        # myclient = pymongo.MongoClient("mongodb://localhost:27017/")
        # mydb = myclient["StatusUpdates"]
        # # mongo_docs = mydb["Users"]
        # statuscol = mydb["StatusUpdates"]
        # mongo_docs = statuscol.find({},{"_id":0})

        with open(filename, mode='r') as statusfile:
            reader = csv.reader(statusfile, delimiter=',')
            status_header = next(reader)
            count = 0
            # print(status_header)
            for line in reader:
                #defadd_status(self, status_id, user_id, status_text):
                # print(f"load {line}")
                main.add_status(line[0], line[1], line[2],status_collection)
                count += 1
                if count == 1000:
                    break
                # print("added:",line[0], line[1], line[2])

        statusfile.close()

    except FileNotFoundError:

        print(f"ERROR: Couldn't find {filename}")

        path = Path(filename)
        print(path)
        return False

    except OSError:
        return False

if __name__ == '__main__':
    socialnetwork_model.User()
    socialnetwork_model.UserStatus()
    myclient = pymongo.MongoClient("mongodb://localhost:27017/")

    mydbs = myclient["StatusUpdates"]
    # mongo_docs = mydb["Users"]
    statusrcol = mydbs["StatusUpdates"]
    statusrcol.drop()

    mydb = myclient["UserAccounts"]
    usercol = mydb["UserAccounts"]
    usercol.drop()
    socialnetwork_model.UserStatus()

    status_collection = main.init_status_collection()
    user_collection = main.init_user_collection()

    reps = 2
    luser = 'load=load_users("accounts.csv", user_collection)'
    lstatus = 'load=load_status_updates("status_updates.csv",status_collection)'

    time.sleep(1)

    load_1000_users = timer(stmt=luser,globals=globals(),number=reps)
    time.sleep(2)
    load_1000_status = timer(stmt=lstatus,globals=globals(),number=reps)
    # print(f"user mango load  {load_1000_users}")
    # print(f"Status MANGO load {load_1000_status}")
    time.sleep(2)
    susers = 'search_1000_users("accounts.csv", user_collection)'
    sstatus = 'search_1000_status_updates("status_updates.csv",status_collection)'
    search_1k_users = timer(stmt=susers,globals=globals(),number=reps)
    time.sleep(1)
    print(f"time to search mNGO 1000 users: , {search_1k_users}")
    search_1k_updates = timer(stmt=sstatus,globals=globals(),number=reps)
    print(f"time to search mango 1000 users: , {search_1k_users}")
    time.sleep(1)
    print(f"time to search mango 1000 updates , {search_1k_updates}")
    time.sleep(1)
    print(f"user mango load , {load_1000_users}")
    print(f"Status mango load , {load_1000_status}")
    add="add_random_users()"
    add5kusers =  timer(stmt=add,globals=globals(),number=reps)
    adds = "add_random_status()"
    add5kstatus =  timer(stmt=adds,globals=globals(),number=reps)

    dels = 'delete_status_updates("status_updates.csv",status_collection)'
    #deletestatus = timer(stmt=dels,globals=globals(),number=1)

    print(f"time to add 5k random user mango:,  {add5kusers}")
    print(f'tile to add random status mango: , {add5kstatus}')
    delu = 'delete_1000_users("accounts.csv", user_collection)'
    deletusers =  timer(stmt=delu,globals=globals(),number=reps)
    print(f"time to delete status mango: , {timer(stmt=dels,globals=globals(),number=1)}")
    print(f"time to delete user mango: , {deletusers}")


import os
import logging
"""
读取文件函数
"""
# ------------------------------

logging.basicConfig(
    level=logging.DEBUG,
    format="\033[37m[%(asctime)s] [%(pathname)s] (%(levelname)s-第%(lineno)d行) \n%(message)s\033[0m")

# ------------------------------

def open(self, all):  # 读取文件夹内文件  # 输入路径
    """
    :param self: 文件夹的路径
    :param all: all==0:是xls文件;all==1:是所有文件
    :return:输出所有文件名的列表
    """
    try:

        file_list = os.walk(self)
        #print(file_list)
        path_list=[]
        count = 0
        for ph in file_list:
            #print(ph)
            ph[0]==self
            path_list = ph[2]

            break
            #count += len(path_list)   # 计数器

        while len(path_list) == 0:
            raise Exception("错误路径或者该文件夹为空")

        a = []
        n = 0
        for e in path_list:
            if all == 1:
                # print("读取文件", '%s' % n, e)
                e = self + e
                a.append(e)

            if all == 0 and ".xls" in e:
                # print("读取文件", '%s' % n, e)
                e = self + e
                a.append(e)

            n = n + 1

        return a

    except Exception as e:
        raise
        #raise e("The folder is empty")
        logging.info(e)
        print("Path error")
        return []

if __name__ == '__main__':
    a=open('E:/HUMEI/核价/',all=1)
    print(a)


import logging
import os

import click
import pandas as pd
import tqdm
from mol_dyn.pipeline import pipeline
from mol_dyn.utils import load_smiles_csv

@click.command()
@click.option("--smiles_csv", required=True, help="Smiles csv")
@click.option("--output_folder", required=True, help="Output folder")
@click.option("--field", type=str, default="pcff", help="Force field")
@click.option("--steps", type=int, default=500000, help="Steps")
@click.option("--cores", type=int, default=8, help="Cores")
@click.option(
    "--emc_template",
    type=str,
    default="mol_dyn/emc_template.esh",
    help="Path to the emc template file in the mol_dyn folder",
)
@click.option(
    "--lammps_template",
    type=str,
    default="mol_dyn/lammps_template.in",
    help="Path to the lammps `template file in the mol_dyn folder",
)
def main(
    smiles_csv: str,
    output_folder: str,
    field: str = "pcff",
    steps: int = 500000,
    cores: int = 8,
    emc_template: str = "mol_dyn/emc_template.esh",
    lammps_template: str = "mol_dyn/lammps_template.in",
):
    logging.basicConfig(level="INFO")

    smiles_data = load_smiles_csv(smiles_csv)
    os.makedirs(output_folder, exist_ok=True)

    status_dict = dict()
    for i in tqdm.tqdm(range(len(smiles_data))):
        # index corresponds to the indice of the row in the smiles csv. The scripts creates a folder for each index, making it easy to correlate the smiles to the simulation folder
        index, smiles = smiles_data.iloc[i]["Indices"], smiles_data.iloc[i]["Smiles"]

        folder_path = os.path.join(output_folder, str(index))
        os.makedirs(folder_path, exist_ok=True)

        status = pipeline(
            folder_path, steps, cores, field, smiles, emc_template, lammps_template
        )

        if not status["gen_emc_file"]:
            logging.warning(
                f"Simulation for index {index} and SMILES {smiles} failed to generate emc input file."
            )
        elif not status["run_emc"]:
            logging.warning(
                f"Simulation for index {index} and SMILES {smiles} failed to run emc."
            )
        elif not status["gen_lammps_file"]:
            logging.warning(
                f"Simulation for index {index} and SMILES {smiles} failed to generate lammps input file."
            )
        elif not status["run_lammps"]:
            logging.warning(
                f"Simulation for index {index} and SMILES {smiles} failed to run lammps."
            )
        elif not status["gen_spectrum"]:
            logging.warning(
                f"Simulation for index {index} and SMILES {smiles} failed to create spectrum from simulation."
            )

        status_dict[index] = status

    status_df = pd.DataFrame.from_dict(status_dict, orient="index")

    logging.info(
        "Success: {} of total {}: {:.3f}%".format(
            status_df["gen_spectrum"].sum(),
            len(status_df),
            status_df["gen_spectrum"].sum() / len(status_df) * 100,
        )
    )
    status_df.to_csv(os.path.join(output_folder, "status_lammps.csv"))

if __name__ == "__main__":
    main()


# -*- coding:utf-8 -*-
import pandas as pd                    # for data handling
import numpy as np                     # for random selections, mainly
import matplotlib.pyplot as plt        # for plotting
import matplotlib
from sklearn.datasets.samples_generator import make_blobs
from sklearn.tree import DecisionTreeClassifier
from baggingPU import BaggingClassifierPU
from sklearn.ensemble import RandomForestClassifier

matplotlib.rcParams['font.sans-serif'] = ['SimHei']   # 用黑体显示中文
matplotlib.rcParams['axes.unicode_minus'] = False     # 正常显示负号

plt.rcParams['figure.figsize'] = 7, 7   # graph dimensions
plt.rcParams['font.size'] = 14         # graph font size

# 0.create datasets of blobs 构建blob是数据集，用于测试PU-Learning不同Approaches的效果
X, y = make_blobs(
    n_samples=6000,
    centers=[[1, 5], [5, 1], [0, 0], [6, 6]]
)
y = (y > 1).astype(int)
X = pd.DataFrame(X, columns=['feature1', 'feature2'])
y = pd.Series(y)

# Check the contents of the set
print('%d data points and %d features' % X.shape)
print('%d positive out of %d total' % (sum(y), len(y)))

# 0.1 plot the original data set 绘制原始blobs数据集
plt.scatter(
    X[y == 0].feature1, X[y == 0].feature2,
    c='k', marker='.', linewidths=1, s=10, alpha=0.5,
    label='Negative'
)

plt.scatter(
    X[y == 1].feature1, X[y == 1].feature2,
    c='b', marker='o', linewidth=0, s=50, alpha=0.5,
    label='Positive'
)
plt.legend()
plt.title('图1- 原始样本数据')
plt.show()

# 0.2 对标记为1的样本取消其样本label 取消数量为hidden_size=2700

y_orig = y.copy()

hidden_size = 2700
y.loc[
    np.random.choice(
        y[y == 1].index,
        replace=False,
        size=hidden_size
    )
] = 0
# Check the new contents of the set
print('%d positive out of %d total' % (sum(y), len(y)))
# plot the data set, as model can see it
plt.scatter(
    X[y == 0].feature1, X[y == 0].feature2,
    c='k', marker='.', linewidths=1, s=10, alpha=0.5,
    label='Unlabeled'
)

plt.scatter(
    X[y == 1].feature1, X[y == 1].feature2,
    c='b', marker='o', linewidth=0, s=50, alpha=0.5,
    label='Positive'
)
plt.legend()
plt.title('图-2 设置2700个positive样本 为 Unlabeled样本 ')
plt.show()

# PU-Learning Approaches 对比
# 1.1 standard classifier 标准分类器
# 使用标准分类器对Blobs样本进行分类
forest = RandomForestClassifier(
    n_estimators=100,
    n_jobs=-1
)

# X 为Blobs的特征数据（共两维）
# y 为Blob数据的标签（已设置2700 positive样本数据集为 unlabeled数据）
forest.fit(X, y)

results = pd.DataFrame({
    'truth': y_orig,
    'label': y,
    'output_std': forest.predict_proba(X)[:, 1]
}, columns=['truth', 'label', 'output_std'])

# 1.2  conclusion of standard classifier 展示`lable==0` 即`unlabeled`的全部数据，查看经过`standard classifier`发现的`positive`的样本
plt.rcParams['figure.figsize'] = 9, 7
plt.scatter(
    X[y == 0].feature1, X[y == 0].feature2,
    c=results[y == 0].output_std, linewidth=0, s=50, alpha=0.5, cmap='jet_r'
)
plt.colorbar(label='Unlabeled样本的预测分值')
plt.title('Standard Classifier')
plt.show()

# 2.1 Bagging
# - Bagging方法

n_estimators = 1000
estimator = DecisionTreeClassifier()

# iP表示为positive样本的index
iP = y[y > 0].index

# iU表示为 unlabeled样本的index
iU = y[y <= 0].index

# 对于每次的数据迭代，记录数据被OOB的次数
num_oob = pd.DataFrame(np.zeros(shape=y.shape), index=y.index)

# 记录每一轮 OOB data的score
sum_oob = pd.DataFrame(np.zeros(shape=y.shape), index=y.index)

for _ in range(n_estimators):
    if _ % 100 == 0:
        print("当前迭代次数为：%d" % _)

    # Bootstrap 抽取数据  每次抽样unlabeled样本数量与positive样本数量比为1：1
    # iB 表示 bagging data的index
    iB = np.random.choice(iU, replace=True, size=len(iP))

    # out of bag样本的index i_oob（unlabeled samples）
    i_oob = list(set(iU) - set(iB))

    # 组合 positive样本集 + unlabeled样本集
    Xb = X[y > 0].append(X.loc[iB])
    yb = y[y > 0].append(y.loc[iB])

    # 训练分类器模型
    estimator.fit(Xb, yb)

    # 迭代过程中循环更新 oob数据的 sum_oob 以及 num_oob
    sum_oob.loc[i_oob, 0] += estimator.predict_proba(X.loc[i_oob])[:, 1]
    num_oob.loc[i_oob, 0] += 1

# 取sum_oob / num_oob 数据值作为 Bagging方法 对unlabeled数据的learning 结果
results['output_bag'] = sum_oob / num_oob

# 2.2 conclusion - PU-Learning bagging的数据 可视化展示`label=0`即 `unlabeled` 的数据
# 展示unlabeled数据集 在经过Bagging方法后的效果
plt.scatter(
    X[y == 0].feature1, X[y == 0].feature2,
    c=results[y == 0].output_bag, linewidth=0, s=50, alpha=0.5, cmap='jet_r'
)
plt.colorbar(label='Unlabeled样本的预测分值')
plt.title('PU Bagging')
plt.show()

# 3.1 Using `BaggingClassifierPU`
bc = BaggingClassifierPU(
    DecisionTreeClassifier(),
    n_estimators=1000,
    max_samples=sum(y),
    n_jobs=-1
)
bc.fit(X, y)
results['output_skb'] = bc.oob_decision_function_[:, 1]
# Visualize the approach's result
plt.scatter(
    X[y == 0].feature1, X[y == 0].feature2,
    c=results[y == 0].output_skb, linewidth=0, s=50, alpha=0.5, cmap='jet_r'
)
plt.colorbar(label='Scores given to unlabeled points')
plt.title(r'Using ${\tt BaggingClassifierPU}$')
plt.show()

# 4. Using `Two-Step`
# 4.1 Step1 + Step2
# Step1 识别可靠负样本集 RN `Reliable Negative`
# 如果未标注数据 得到score超过全部已知的positive样本数据的score值，则label为positive
# 或者score低于全部已知的positive的score值 ，则label为negative
# Step2 利用 `Positive Samples` 和 `RN`集合组成训练集，训练传统的二分类模型

# 对label进行调整
# `1` 表示为`positive`;
# `-1` 表示为`unlabeled `;
# `0` 表示为实锤`negative`;

ys = 2 * y - 1

# 根据普遍意义的随机森林预测样本数据
pred = forest.predict_proba(X)[:, 1]

# 根据预测结果 判断 positive 数据的概率范围
range_P = [min(pred * (ys > 0)), max(pred * (ys > 0))]

# STEP 1
# 如果未标注数据 得到score超过全部已知的positive样本数据的score值，则label为postive
# 或者score低于全部已知的positive的score值 ，则label为negative

# 新的positive样本
iP_new = ys[(ys < 0) & (pred >= range_P[1])].index
# 新的negative样本
iN_new = ys[(ys < 0) & (pred <= range_P[0])].index

# 实锤unlabeled样本数据的标签
ys.loc[iP_new] = 1
ys.loc[iN_new] = 0

# STEP1通过散点图查看标注的效果
plt.rcParams['figure.figsize'] = 7, 7
cdict = {-1: 'gray', 0: 'darkred', 1: 'blue'}

plt.scatter(
    X.feature1, X.feature2,
    c=[cdict[k] for k in ys],
    linewidths=0, s=20, alpha=0.5
)

plt.title('经过Step-1后样本的标记情况图')
plt.show()

# Classifier to be used for step 2
rf2 = RandomForestClassifier(n_estimators=1000, n_jobs=-1)

# Limit to 10 iterations (this is arbitrary, but
# otherwise this approach can take a very long time)
# 随机设置超参数迭代次数为10次
for i in range(10):
    # If step 1 didn't find new labels, we're done
    if len(iP_new) + len(iN_new) == 0 and i > 0:
        break

    print('Step 1 labeled %d new positives and %d new negatives.' % (len(iP_new), len(iN_new)))
    print('正在执行Step2，当前迭代次数为%d次' % (i + 1), end=' ')

    # STEP 2
    # Retrain on new labels and get new scores
    rf2.fit(X, ys)
    pred = rf2.predict_proba(X)[:, -1]

    # Find the range of scores given to positive data points
    range_P = [min(pred * (ys > 0)), max(pred * (ys > 0))]

    # Repeat step 1
    iP_new = ys[(ys < 0) & (pred >= range_P[1])].index
    iN_new = ys[(ys < 0) & (pred <= range_P[0])].index
    ys.loc[iP_new] = 1
    ys.loc[iN_new] = 0

# Lastly, get the scores assigned by this approach
results['output_stp'] = pred

plt.scatter(
    X.feature1, X.feature2,
    c=[cdict[k] for k in ys],
    linewidths=0, s=20, alpha=0.5
)

plt.title('Step-2处理迭代完成后样本的标记情况图')
plt.show()

# 4.2 对Two-Step方法的效果可视化展示
# Visualize this approach's final results
plt.rcParams['figure.figsize']=9,7
plt.scatter(
    X[y == 0].feature1, X[y == 0].feature2,
    c=results[y == 0].output_stp, linewidths=0, s=50, alpha=0.5, cmap='jet_r'
)
plt.colorbar(label='Scored given to unlabeled points')
plt.title('Two-Step method PU Learning')
plt.show()

# 5. 方法比较
# 对于每一个数据，计算3种方法的平均值
results['output_all'] = results[['output_std', 'output_bag', 'output_stp']].mean(axis=1)
# 5.1 可视化三种不同方法的平均效果
plt.rcParams['figure.figsize'] = 9, 7
plt.scatter(
    X[y == 0].feature1, X[y == 0].feature2, c=results[y==0].output_all, linewidths=0, s=50, alpha=0.5
)
plt.colorbar(label='Unlabeled样本的预测分值')
plt.title('PU-Learning三种不同方法的平均效果')
plt.show()

# 5.2 对于三种不同的PU-Learning的表现可视化展示  -成功识别隐藏positive样本的数量
# 打印经过output_std learning 方法后，排名前十的unlabeled数据的真实label标签
results[results.label==0].sort_values('output_std', ascending=False).head(10).truth
# 将全部hidden_size 按照步长为100设置不同的数量级
ts = range(100, hidden_size, 100)
y_std, y_bag, y_skb, y_stp, y_all = [], [], [], [], []

for t in ts:
    # 针对不同的PU-Learning的方法添加
    # 对未标注数据的识别的positive分数最高的TOP(100,200,300,....3000) 真正positive样本的平均值

    # 方法一: standard classifier
    y_std.append(
        results[results.label == 0].sort_values(
            'output_std', ascending=False
        ).head(t).truth.mean()
    )
    # 方法二: Bagging
    y_bag.append(
        results[results.label == 0].sort_values(
            'output_bag', ascending=False
        ).head(t).truth.mean()
    )
    # 方法三：BaggingClassifierPU
    y_skb.append(
        results[results.label == 0].sort_values(
            'output_skb', ascending=False
        ).head(t).truth.mean()
    )
    # 方法四：TWO-STEP
    y_stp.append(
        results[results.label == 0].sort_values(
            'output_stp', ascending=False
        ).head(t).truth.mean()
    )
    # 方法五：方法一、二、四比较的平均值
    y_all.append(
        results[results.label == 0].sort_values(
            'output_all', ascending=False
        ).head(t).truth.mean()
    )

# 比较PU-Bagging和BaggingClassifierPU 的区别
print([y_bag[i] - y_skb[i] for i in range(len(y_bag))])

# - 四种不同的PU-learning的处理方法展示 -- Performance graphing
plt.rcParams['font.size'] = 16
plt.rcParams['figure.figsize'] = 15, 8

plt.plot(
    ts, y_std,
    ts, y_bag,
    ts, y_stp,
    ts, y_all,
    lw=3
)
vals = plt.gca().get_yticks()
plt.yticks(vals, ['%.0f%%' %(v*100) for v in vals])
plt.xlabel('按照最高分数排序选择的未标注数据量')
plt.ylabel('被选中样本肯定为positive的百分比')
plt.legend([
    'Standard classifier',
    'PU Bagging',
    'Two-Step approach',
    'Average score'
])

ylim = plt.gca().get_ylim()
plt.title('三种PU-Learning的方法以及其平均值的表现图')
plt.grid()
plt.show()


#!/usr/bin/env python3

import os
import click
import hashlib
import imageio
from .jpeg import *
import subprocess as sp
from pathlib import Path
from contextlib import suppress
from multiprocessing import cpu_count

import concurrent.futures

cache_dir = Path.home() / '.cache/glitch-art-display'
cache_dir.mkdir(exist_ok=True)

frame_timings = [5, 4, 3, 2, 2, 1, 1, 1, 1]

def is_cached(filename):

    if (cache_dir / filename).is_file():
        return True
    return False

def find_images(directory):

    valid_suffixes = ['.jpg', '.jpeg']
    convert_suffixes = ['.png']

    for root,dirs,files in os.walk(directory):
        for file in files:
            filename = Path(root) / file

            # if image is a JPEG
            if any([filename.suffix.lower().endswith(s) for s in valid_suffixes]):
                yield filename

            # if image is an unsupported image type
            elif any([filename.suffix.lower().endswith(s) for s in convert_suffixes]):
                # try to convert it with imagemagick
                try:
                    new_filename = cache_dir / (filename.stem + '.jpg')
                    sp.run(['convert', str(filename), str(new_filename)], check=True)
                    yield new_filename
                except (FileNotFoundError, sp.CalledProcessError) as e:
                    print(f'[!] Unsupported file: {filename.name}')
                    print(f'[!]  - please install imagemagick in order to use {filename.suffix} files')

def gen_frames(image_dir, output, glitch_amount=100, fps=25, num_image_frames=25, num_transition_frames=30, shuffle=False):

    frame_groups = []
    frames_output = Path(output).resolve()
    frames_output.parent.mkdir(parents=True, exist_ok=True)
    frame_number = 0

    transition_frames = int(num_transition_frames/2)
    found_images = list(find_images(image_dir))
    if shuffle:
        random.shuffle(found_images)

    with concurrent.futures.ThreadPoolExecutor(max_workers=cpu_count()) as pool:
        for image in found_images:
            try:
                print(f'[+] Generating frames for {image.name}')

                glitched_frames = []

                image_bytes = bytearray(image.read_bytes())
                image_hash = hashlib.md5(image_bytes).hexdigest()
                png_filename = cache_dir / f'{image_hash}.png'
                if not png_filename.exists():
                    pool.submit(sp.run, ['convert', str(image), str(png_filename)], check=True)

                # generate glitch frames
                glitched_futures = []
                for i in range(transition_frames):
                    amount = int(glitch_amount * ((i+1)/transition_frames))
                    glitched_futures.append(pool.submit(glitch, image, amount, i))

                concurrent.futures.wait(glitched_futures)
                for glitched_future in glitched_futures:
                    glitched_frame = glitched_future.result()
                    if glitched_frame:
                        glitched_frames.append(glitched_frame)
            except KeyboardInterrupt:
                pool.shutdown(wait=False, cancel_futures=True)
                raise

            glitch_in = []
            random.shuffle(frame_timings)
            for i, glitched_image in enumerate(glitched_frames[::-1]):
                glitch_group = []
                for j in range(frame_timings[i % len(frame_timings)]):
                    glitch_group.append(glitched_image)
                glitch_in.append(glitch_group)

            normal = []
            for i in range(num_image_frames):
                normal.append(png_filename)

            glitch_out = []
            random.shuffle(frame_timings)
            for i, glitched_image in enumerate(glitched_frames):
                glitch_group = []
                for j in range(frame_timings[i % len(frame_timings)]):
                    glitch_group.append(glitched_image)
                glitch_out.append(glitch_group)

            frame_groups.append([glitch_in, normal, glitch_out])

    frames = []
    interlace_frames = max(2, min(int(transition_frames/4), 1))
    for i, [glitch_in, normal, glitch_out] in enumerate(frame_groups):
        for g in glitch_in:
            frames += g
        frames += normal
        if i == len(frame_groups)-1:
            for g in glitch_out:
                frames += g
        else:
            # truncate end of this glitch group
            glitch_out, glitch_interlaced = glitch_out[:-interlace_frames], glitch_out[-interlace_frames:]

            # truncate beginning of next glitch group
            frame_groups[i+1][0], glitch_next_interlaced = frame_groups[i+1][0][interlace_frames:], frame_groups[i+1][0][:interlace_frames]

            for g in glitch_out:
                frames += g
            for i in range(min(interlace_frames, len(glitch_interlaced))):
                frames += glitch_next_interlaced[i]
                frames += glitch_interlaced[i]

    w = imageio.get_writer(str(frames_output), format='FFMPEG', mode='I', fps=fps, macro_block_size=1)
    try:
        prev_frame = ''
        for i, frame in enumerate(frames):
            print(f'\r[+] Rendering frame {i:,}/{len(frames):,} ({i/len(frames)*100:.1f}%)', end='')
            frame = str(frame)
            if frame != prev_frame:
                try:
                    read_frame = imageio.imread(frame, pilmode='RGB')
                except Exception as e:
                    print(f'[!] Error reading {frame}: {e}')
                    continue
            try:
                w.append_data(read_frame)
            except ValueError:
                print(f'[!] {frame}')
                raise
            prev_frame = str(frame)
    finally:
        with suppress(Exception):
            w.close()

    print(f'[+] Video saved to {frames_output}')

def glitch(image, amount=None, sequence=''):

    print(f'[+] Glitching {image.name} by {amount}')

    if amount is None:
        amount = random.randint(0,99)
    else:
        amount = max(0, min(99, int(amount)-1))

    image = Path(image)

    image_bytes = bytearray(image.read_bytes())
    image_hash = hashlib.md5(image_bytes).hexdigest()
    try:
        jpeg = Jpeg(image_bytes)
    except JpegError as e:
        print(f'[!] {e}')
        return

    jpeg_filename = cache_dir / f'{image_hash}_{amount}_{sequence}.jpg'
    png_filename = cache_dir / f'{image_hash}_{amount}_{sequence}.png'

    print(f'[+] Glitching {image.name} by {amount}')

    # checked for cached files
    if is_cached(png_filename):
        print(f'[+] Found cached frame for {png_filename}')

    else:
        while 1:
            try:
                jpeg.amount      = amount
                jpeg.seed        = random.randint(0,99)
                jpeg.iterations  = max(0, min(115, int(amount*1.15)))

                # create a new image if not cached
                jpeg.save_image(jpeg_filename)
                try:
                    sp.run(['convert', str(jpeg_filename), str(png_filename)], check=True)
                    break
                except Exception:
                    continue

            except JpegError as e:
                print(f'[!] {e}')
                continue

    return png_filename

def main():
    @click.command(context_settings={'show_default': True})
    @click.argument('input', required=True, type=click.Path(exists=True, file_okay=False, dir_okay=True))
    @click.argument('output', required=True, type=click.Path(dir_okay=False))
    @click.option('--amount', default=50, type=click.IntRange(1, 100, clamp=True), help='Glitch amount')
    @click.option('--fps', default=25, type=float, help='Frames per second')
    @click.option('--normal-frames', default=25*25, type=int, help='Number of normal frames')
    @click.option('--transition-frames', default=30, type=int, help='Number of glitchy transition frames')
    @click.option('--dont-shuffle', is_flag=True, help='Don\'t shuffle order of images')
    def go(input, output, amount, fps, normal_frames, transition_frames, dont_shuffle):
        gen_frames(input, output, amount, fps, normal_frames, transition_frames, not dont_shuffle)

    try:
        go()
    except KeyboardInterrupt:
        print("Interrupted")

if __name__ == '__main__':
    main()


#!/usr/bin/python
# -*- coding: utf-8 -*-
from os import listdir,path,curdir
from os.path import isfile, join
import re
import shutil
import pandas as pd
from sklearn.model_selection import train_test_split
import csv
import random

root = "/data01/ML/dataset/FACE_CLASSIFIER"
face_image_path = "/data01/ML/dataset/FACE_CLASSIFIER/CelebA2/CelebA/Img/img_celeba/img_celeba/img_celeba"
objects_image_path = "/data01/ML/dataset/FACE_CLASSIFIER/256_ObjectCategories_modified"
face_image_path2 = "/data01/ML/dataset/FACE_CLASSIFIER/Borderline"

def get_faces(balanced, face_image_path, face_image_path2):
    filenames = []
    classification = []
    files = [f for f in listdir(face_image_path) if
             isfile(join(face_image_path, f))]
    files = random.sample(files, balanced-600)
    for f in files:
        filenames.append(face_image_path + "/" + f)
        classification.append(1)
    files2 = [f for f in listdir(face_image_path2) if
             isfile(join(face_image_path2, f))]
    files2 = random.sample(files2, 600)
    for f in files2:
        filenames.append(face_image_path2 + "/" + f)
        classification.append(1)
    return filenames, classification

def get_balanced_data_from_classes(min,objects_main_folder):
    filenames = []
    classification = []
    dirs = [f for f in listdir(objects_main_folder) if not isfile(join(objects_main_folder, f))]
    for dir_ in dirs:
        files = [f for f in listdir(objects_main_folder + "/" + dir_) if
                 isfile(join(objects_main_folder + "/" + dir_, f))]
        files = random.sample(files, min)
        for f in files:
            filenames.append(objects_main_folder+"/"+dir_+"/"+f)
            classification.append(0)
    return filenames, classification

def get_not_balanced_data_from_classes(objects_main_folder):
    filenames = []
    classification = []
    dirs = [f for f in listdir(objects_main_folder) if not isfile(join(objects_main_folder, f))]
    for dir_ in dirs:
        files = [f for f in listdir(objects_main_folder + "/" + dir_) if
                 isfile(join(objects_main_folder + "/" + dir_, f))]
        for f in files:
            filenames.append(objects_main_folder + "/" + dir_ + "/" + f)
            classification.append(0)
    return filenames, classification

def count_min_in_objects_types(objects_main_folder):
    dirs = [f for f in listdir(objects_main_folder) if not isfile(join(objects_main_folder, f))]
    min_f_numbers = 1000000
    max_f_numbers = 0
    for dir_ in dirs:
        files = [f for f in listdir(objects_main_folder+"/"+dir_) if isfile(join(objects_main_folder+"/"+dir_, f))]
        if len(files)< min_f_numbers:
            min_f_numbers = len(files)
        if len(files) > max_f_numbers:
            max_f_numbers = len(files)
    return min_f_numbers

def get_min_between_celeba_and_objects(objects_main_folder, celeba_images_folder):
    dirs = [f for f in listdir(objects_main_folder) if not isfile(join(objects_main_folder, f))]
    count_objects = 0
    count_faces = 0
    for dir_ in dirs:
        files = [f for f in listdir(objects_main_folder + "/" + dir_) if
                 isfile(join(objects_main_folder + "/" + dir_, f))]
        count_objects = count_objects + len(files)
    files_Faces = [f for f in listdir(celeba_images_folder) if
                   isfile(join(celeba_images_folder, f))]
    count_faces = count_faces + len(files_Faces)
    print(count_faces, count_objects)
    return count_faces if count_faces < count_objects else count_objects

def write_csv(category, non_faces, faces):
    with open("./dataset/" + category+".csv" , mode='w', newline='') as file:
        file = csv.writer(file, delimiter=',', quotechar='"', quoting=csv.QUOTE_MINIMAL)
        file.writerow(['image_path', 'face'])
        for el in non_faces:
            file.writerow([el[0], el[1]])
        for el in faces:
            file.writerow([el[0], el[1]])

if __name__ == '__main__':
    #min = count_min_in_objects_types(objects_image_path)
    #non_faces = get_balanced_data_from_classes(min,objects_image_path)
    min = get_min_between_celeba_and_objects(objects_image_path, face_image_path)
    non_faces = get_not_balanced_data_from_classes(objects_image_path)
    print(len(non_faces[0]))
    #print(non_faces[1])
    non_faces_x_train, non_faces_x_test, non_faces_y_train, non_faces_y_test = train_test_split(non_faces[0], non_faces[1], test_size = 0.15,shuffle=True)
    non_faces_x_train, non_faces_x_val, non_faces_y_train, non_faces_y_val = train_test_split(non_faces_x_train, non_faces_y_train, test_size= 0.15,shuffle=True)
    print(len(non_faces_x_train),len(non_faces_x_val), len(non_faces_x_test))
    #for el in zip(non_faces_x_train, non_faces_y_train):
    #    print(el)

    faces = get_faces(len(non_faces[0]), face_image_path, face_image_path2)
    print(len(faces[0]))
    #print(faces[1])
    faces_x_train, faces_x_test, faces_y_train, faces_y_test = train_test_split(faces[0],faces[1],test_size=0.15,shuffle=True)
    faces_x_train, faces_x_val, faces_y_train, faces_y_val = train_test_split(faces_x_train,faces_y_train,
                                                                                test_size=0.15,shuffle=True)
    print(len(non_faces_x_train), len(non_faces_x_val), len(non_faces_x_test))
    #Define true dataset
    non_faces_train, faces_train = zip(non_faces_x_train,non_faces_y_train),zip(faces_x_train,faces_y_train)
    non_faces_val, faces_val = zip(non_faces_x_val,non_faces_y_val),zip(faces_x_val,faces_y_val)
    non_faces_test, faces_test = zip(non_faces_x_test,non_faces_y_test),zip(faces_x_test,faces_y_test)

    for tuple in [("train3", non_faces_train,faces_train),("val3",non_faces_val,faces_val),("test3", non_faces_test,faces_test)]:
         print("Writing:", tuple[0])
         write_csv(tuple[0],tuple[1],tuple[2])
    # with open('C:/Users/Lorenzo/Desktop/Università/Machine_learning_project/dataset/FACE_CLASSIFIER/dataset.csv' , mode='w', newline='') as file:
    #     file = csv.writer(file, delimiter=',', quotechar='"', quoting=csv.QUOTE_MINIMAL)
    #     file.writerow(['image_path', 'face'])
    #     for el in non_faces:
    #         file.writerow([el[0], el[1]])
    #     for el in faces:
    #         file.writerow([el[0], el[1]])


import sklearn
import joblib
import cv2
import torch
import torch.nn as nn
import torch.nn.functional as F
import numpy as np
import albumentations
from torch.utils.data import Dataset, DataLoader
from PIL import Image
import time
import firebase_admin
from firebase_admin import credentials
from firebase_admin import firestore
cred = credentials.Certificate("pooleye-68d49-firebase-adminsdk-zkgaj-4f20e306ba.json")
firebase_admin.initialize_app(cred)
from playsound import playsound
import warnings
warnings.filterwarnings("ignore")

lb = joblib.load('lb.pkl')
class CustomCNN(nn.Module):
    def __init__(self):
        super(CustomCNN, self).__init__()
        self.conv1 = nn.Conv2d(3, 16, 5)
        self.conv2 = nn.Conv2d(16, 32, 5)
        self.conv3 = nn.Conv2d(32, 64, 3)
        self.conv4 = nn.Conv2d(64, 128, 5)
        self.fc1 = nn.Linear(128, 256)
        self.fc2 = nn.Linear(256, len(lb.classes_))
        self.pool = nn.MaxPool2d(2, 2)
    def forward(self, x):
        x = self.pool(F.relu(self.conv1(x)))
        x = self.pool(F.relu(self.conv2(x)))
        x = self.pool(F.relu(self.conv3(x)))
        x = self.pool(F.relu(self.conv4(x)))
        bs, _, _, _ = x.shape
        x = F.adaptive_avg_pool2d(x, 1).reshape(bs, -1)
        x = F.relu(self.fc1(x))
        x = self.fc2(x)
        return x

print('Loading model and label binarizer...')
lb = joblib.load('lb.pkl')
model = CustomCNN().cuda()
print('Model Loaded...')
model.load_state_dict(torch.load('model.pth'))
print('Loaded model state_dict...')
aug = albumentations.Compose([
    albumentations.Resize(224, 224),
    ])

class detection:
  countDrowning=0
  ThersholdForDrowning =  30;
  fram=0
  def detectDrowning(self):
    #input from the camera
    cap = cv2.VideoCapture(0)
    if (cap.isOpened() == False):
        print('Error while trying to read video')
    frame_width = int(cap.get(3))
    frame_height = int(cap.get(4))
    while(cap.isOpened()):
        start_time = time.time()
        ret, frame = cap.read()
        if ret == True:
            model.eval()
            with torch.no_grad():
                pil_image = Image.fromarray(cv2.cvtColor(frame, cv2.COLOR_BGR2RGB))
                pil_image = aug(image=np.array(pil_image))['image']
                if self.fram == 500:
                 break
                self.fram+=1
                pil_image = np.transpose(pil_image, (2, 0, 1)).astype(np.float32)
                pil_image = torch.tensor(pil_image, dtype=torch.float).cuda()
                pil_image = pil_image.unsqueeze(0)
                outputs = model(pil_image)
                _, preds = torch.max(outputs.data, 1)
            self.calculateDrowning(lb.classes_[preds])
            print("Frame classified as: ",lb.classes_[preds])
        else:
            break

        print("FPS: ", int(1 / (time.time() - start_time)))

    #print(self.countDrowning)
    return 0

  def calculateDrowning(self,classClassified):
    if classClassified == 'drowning':
      self.countDrowning +=1;
      if self.countDrowning >= self.ThersholdForDrowning:
        playsound('alarm.mp3')
        self.alertDrowning()
        self.countDrowning = 0;
    else:
      	self.countDrowning = 0;

  def alertDrowning(self):
    organizationId = "25771623413315686"
    sent = False
    text = "drowning alert"
    db = firestore.client()
    collectionBane = db.collection("lifeguardnotifications").add({
      "orgID": organizationId,
      "sent": sent,
      "text": text,
      "sentTO":"lifeguard"
    })

d = detection()
d.detectDrowning()


# -*- coding: utf-8 -*-
"""Classes for lidar ratio related db tables"""

from ELDAmwl.database.tables.db_base import Base
from sqlalchemy import Column
from sqlalchemy import DECIMAL
from sqlalchemy import INTEGER
from sqlalchemy import text

class ExtBscOption(Base):
    """content of the db table ext_bsc_options

    """

    __tablename__ = 'ext_bsc_options'

    ID = Column(
        INTEGER,
        primary_key=True,
    )
    product_id = Column(
        '_product_ID',
        INTEGER,
        nullable=False,
        server_default=text("'-1'"),
    )
    extinction_options_product_id = Column(
        '_extinction_options_product_ID',
        INTEGER,
        nullable=False,
        index=True,
        server_default=text("'-1'"),
    )
    raman_backscatter_options_product_id = Column(
        '_raman_backscatter_options_product_ID',
        INTEGER,
        nullable=False,
        index=True,
        server_default=text("'-1'"),
    )
    error_method_id = Column(
        '_error_method_ID',
        INTEGER,
        nullable=False,
        index=True,
        server_default=text("'-1'"),
    )
    min_BscRatio_for_LR = Column(
        DECIMAL(10, 4),
        nullable=False,
        server_default=text("'1.0000'"),
    )


# -*- coding: utf-8 -*-
import itertools

from flask import url_for
from flask_mail import Message
import query_phenomizer

from scout.constants import (CASE_STATUSES, PHENOTYPE_GROUPS, COHORT_TAGS)
from scout.models.event import VERBS_MAP
from scout.server.utils import institute_and_case

STATUS_MAP = {'solved': 'bg-success', 'archived': 'bg-warning'}
SEX_MAP = {'1': 'male', '2': 'female'}
PHENOTYPE_MAP = {-9: 'missing', 0: 'missing', 1: 'unaffected', 2: 'affected'}

def cases(store, case_query):
    """Preprocess case objects."""
    case_groups = {status: [] for status in CASE_STATUSES}
    for case_obj in case_query:
        analysis_types = set(ind['analysis_type'] for ind in case_obj['individuals'])
        case_obj['analysis_types'] = list(analysis_types)
        case_obj['assignees'] = [store.user(user_email) for user_email in
                                 case_obj.get('assignees', [])]
        case_groups[case_obj['status']].append(case_obj)

    data = {
        'prio_cases': case_groups['prioritized'],
        'cases': [(status, case_groups[status]) for status in CASE_STATUSES[1:]],
        'found_cases': case_query.count(),
    }
    return data

def case(store, institute_obj, case_obj):
    """Preprocess a single case."""
    for individual in case_obj['individuals']:
        individual['sex_human'] = SEX_MAP.get(individual['sex'], 'unknown')
        individual['phenotype_human'] = PHENOTYPE_MAP.get(individual['phenotype'])

    case_obj['assignees'] = [store.user(user_email) for user_email in
                             case_obj.get('assignees', [])]
    suspects = [store.variant(variant_id) or variant_id for variant_id in
                case_obj.get('suspects', [])]
    causatives = [store.variant(variant_id) or variant_id for variant_id in
                  case_obj.get('causatives', [])]

    distinct_genes = set()
    case_obj['panel_names'] = []
    for panel_info in case_obj.get('panels', []):
        if panel_info.get('is_default'):
            panel_obj = store.panel(panel_info['panel_id'])
            distinct_genes.update([gene['hgnc_id'] for gene in panel_obj['genes']])
            full_name = "{} ({})".format(panel_obj['display_name'], panel_obj['version'])
            case_obj['panel_names'].append(full_name)
    case_obj['default_genes'] = list(distinct_genes)

    for hpo_term in itertools.chain(case_obj.get('phenotype_groups', []),
                                    case_obj.get('phenotype_terms', [])):
        hpo_term['hpo_link'] = ("http://compbio.charite.de/hpoweb/showterm?id={}"
                                .format(hpo_term['phenotype_id']))

    # other collaborators than the owner of the case
    case_obj['o_collaborators'] = [collab_id for collab_id in
                                   case_obj['collaborators'] if
                                   collab_id != case_obj['owner']]

    irrelevant_ids = ('cust000', institute_obj['_id'])
    collab_ids = [collab['_id'] for collab in store.institutes() if
                  (collab['_id'] not in irrelevant_ids) and
                  (collab['_id'] not in case_obj['collaborators'])]

    events = list(store.events(institute_obj, case=case_obj))
    for event in events:
        event['verb'] = VERBS_MAP[event['verb']]

    data = {
        'status_class': STATUS_MAP.get(case_obj['status']),
        'other_causatives': store.check_causatives(case_obj),
        'comments': store.events(institute_obj, case=case_obj, comments=True),
        'hpo_groups': PHENOTYPE_GROUPS,
        'events': events,
        'suspects': suspects,
        'causatives': causatives,
        'collaborators': collab_ids,
        'cohort_tags': COHORT_TAGS,
    }
    return data

def update_synopsis(store, institute_obj, case_obj, user_obj, new_synopsis):
    """Update synopsis."""
    # create event only if synopsis was actually changed
    if new_synopsis and case_obj['synopsis'] != new_synopsis:
        link = url_for('cases.case', institute_id=institute_obj['_id'],
                       case_name=case_obj['display_name'])
        store.update_synopsis(institute_obj, case_obj, user_obj, link,
                              content=new_synopsis)

def hpo_diseases(username, password, hpo_ids, p_value_treshold=1):
    """Return the list of HGNC symbols that match annotated HPO terms.

    Args:
        username (str): username to use for phenomizer connection
        password (str): password to use for phenomizer connection

    Returns:
        query_result: a generator of dictionaries on the form
        {
            'p_value': float,
            'disease_source': str,
            'disease_nr': int,
            'gene_symbols': list(str),
            'description': str,
            'raw_line': str
        }
    """
    # skip querying Phenomizer unless at least one HPO terms exists
    try:
        results = query_phenomizer.query(username, password, *hpo_ids)
        diseases = [result for result in results
                    if result['p_value'] <= p_value_treshold]
        return diseases
    except SystemExit:
        return None

def rerun(store, mail, current_user, institute_id, case_name, sender, recipient):
    """Request a rerun by email."""
    institute_obj, case_obj = institute_and_case(store, institute_id, case_name)
    user_obj = store.user(current_user.email)
    link = url_for('cases.case', institute_id=institute_id, case_name=case_name)
    store.request_rerun(institute_obj, case_obj, user_obj, link)

    # this should send a JSON document to the SuSy API in the future
    html = """

{institute}: {case} ({case_id})

Re-run requested by: {name}

    """.format(institute=institute_obj['display_name'],
               case=case_obj['display_name'], case_id=case_obj['_id'],
               name=user_obj['name'].encode())

    # compose and send the email message
    msg = Message(subject=("SCOUT: request RERUN for {}"
                           .format(case_obj['display_name'])),
                  html=html, sender=sender, recipients=[recipient],
                  # cc the sender of the email for confirmation
                  cc=[user_obj['email']])
    mail.send(msg)


import click
import numpy as np
from sklearn.feature_extraction.text import CountVectorizer, TfidfVectorizer

def get_wiki_vocab(file):
    """
    Build a vocabulary from Wikipedia articles.

    Parameters
    ----------
    file : str
        File path to Wikipedia article text

    Returns
    -------
    dict
        Dictionary of words and their frequencies
    """
    vocab = {}
    with open(file, 'r', encoding='utf-8') as f:
        line = f.readline()
        while line:
            if line[0:3]!='XXX':
                line=line.strip('\n').split()
                for token in line:
                    vocab[token.lower()] = vocab.get(token.lower(), 0) + 1
            line=f.readline()
    return vocab

def get_notes_vocab(file):
    """
    Build a vocabulary from MIMIC notes text.

    Parameters
    ----------
    file : str
        File path to combined_dataset from 1_preprocess_mimic.py

    Returns
    -------
    dict
        Dictionary of words and their frequencies
    """
    vocab = {}
    with open(file, 'r', encoding='utf-8') as f:
        line = f.readline()
        while line:
            line = line.strip('\n').split()
            if line[0] == 'codes:':
                line = f.readline()
                line = line.strip('\n').split()
                if line[0] == 'notes:':
                    line = f.readline()
                    while line != 'end!\n':
                        line = line.strip('\n').split()
                        for token in line:
                            vocab[token.lower()] = vocab.get(token.lower(), 0) + 1
                        line = f.readline()
            line = f.readline()
    return vocab

def docs_to_strings(documents):
    """
    Convert a list of lists of tokens to a list of strings.

    Parameters
    ----------
    documents : list
        List of lists of tokens

    Returns
    -------
    list
        List of strings
    """
    data = []
    for doc in documents:
        text = ''
        for token in doc:
            text += token + ' '
        data.append(text)
    return data

def get_wiki_documents(file, vocab):
    """
    Get a list of lists of tokens from Wikipedia articles.

    Parameters
    ----------
    file : str
        File path to Wikipedia article texts
    vocab : dict
        Dictionary of words and their frequencies

    Returns
    -------
    list
        List of lists of tokens
    """
    documents = []
    with open(file, 'r', encoding='utf-8') as f:
        line = f.readline()
        while line:
            if line[0:4] == 'XXXd':
                doc = []
                line = f.readline()
                while line[0:4] != 'XXXe':
                    line = line.strip('\n').split()
                    for token in line:
                        if token.lower() in vocab:
                            doc.append(token.lower())
                    line = f.readline()
                documents.append(doc)
            line = f.readline()

    return documents

def get_notes(file, vocab):
    """
    Get a list of lists of tokens from MIMIC notes.

    Parameters
    ----------
    file : str
        File path to combined_dataset from 1_preprocess_mimic.py
    vocab : dict
        Dictionary of words and their frequencies

    Returns
    -------
    list
        List of lists of tokens
    """
    documents = []
    with open(file, 'r', encoding='utf-8') as f:
        line = f.readline()
        while line:
            line = line.strip('\n').split()
            if line[0] == 'codes:':
                line = f.readline()
                line = line.strip('\n').split()
                if line[0] == 'notes:':
                    doc = []
                    line = f.readline()
                    while line != 'end!\n':
                        line = line.strip('\n').split()
                        for token in line:
                            if token.lower() in vocab:
                                doc.append(token)
                        line = f.readline()
                    documents.append(doc)
            line = f.readline()

    return documents

def vectorize(data, output_file, vectorizer):
    """
    Apply a vectorizer to a list of strings.

    Parameters
    ----------
    data : list
        List of strings
    output_file : str
        File path to save vectorized data
    vectorizer : sklearn.feature_extraction.text.CountVectorizer or sklearn.feature_extraction.text.TfidfVectorizer
        Vectorizer to apply
    """
    embed = vectorizer.fit_transform(data)
    embed = embed.A
    embed = np.array(embed, dtype=float)
    np.save(output_file, embed)

def process_data(file_wiki,
                 file_mimic,
                 output_wiki,
                 output_mimic,
                 vectorizer_type):
    wiki_vocab = get_wiki_vocab(file_wiki)
    note_vocab = get_notes_vocab(file_mimic)

    vocab = set(note_vocab.keys()).intersection(set(wiki_vocab.keys()))

    wiki_docs = get_wiki_documents(file_wiki, vocab)
    notes = get_notes(file_mimic, vocab)

    vocab_map = {}
    for note in notes:
        for token in note:
            if token.lower() not in vocab_map.keys():
                vocab_map[token.lower()] = len(vocab_map)

    wiki_docs = docs_to_strings(wiki_docs)
    notes = docs_to_strings(notes)

    if vectorizer_type == 'binary':
        vectorizer = CountVectorizer(min_df=1, vocabulary=vocab_map, binary=True)
    elif vectorizer_type == 'count':
        vectorizer = TfidfVectorizer(min_df=1, vocabulary=vocab_map, use_idf=False)
    elif vectorizer_type == 'tfidf':
        vectorizer = TfidfVectorizer(min_df=1, vocabulary=vocab_map)

    vectorize(wiki_docs, output_wiki, vectorizer)
    vectorize(notes, output_mimic, vectorizer)

@click.command()
@click.option('--file_wiki', default='data/wikipedia_knowledge')
@click.option('--file_mimic', default='data/combined_dataset')
@click.option('--output_wiki', default='data/wikivec')
@click.option('--output_mimic', default='data/notevec')
@click.option('--vectorizer_type', default='binary', type=click.Choice(['binary', 'count', 'tfidf']))
def process_data_(file_wiki,
                  file_mimic,
                  output_wiki,
                  output_mimic,
                  vectorizer_type):
    process_data(file_wiki,
                 file_mimic,
                 output_wiki,
                 output_mimic,
                 vectorizer_type)

if __name__ == "__main__":
    process_data_()


import time
from typing import List, Tuple, Iterable

from volatility3.framework import constants, interfaces, layers, symbols
from volatility3.framework.configuration import requirements
from volatility3.framework.interfaces import plugins
from volatility3.framework.renderers import TreeGrid
from volatility3.framework.symbols import intermed
from volatility3.framework.symbols.windows import extensions

class Info(plugins.PluginInterface):
    """Show OS & kernel details of the memory sample being analyzed."""

    _required_framework_version = (2, 0, 0)
    _version = (1, 0, 0)

    @classmethod
    def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]:
        return [
            requirements.ModuleRequirement(
                name="kernel",
                description="Windows kernel",
                architectures=["Intel32", "Intel64"],
            ),
        ]

    @classmethod
    def get_depends(
        cls,
        context: interfaces.context.ContextInterface,
        layer_name: str,
        index: int = 0,
    ) -> Iterable[Tuple[int, interfaces.layers.DataLayerInterface]]:
        """List the dependencies of a given layer.

        Args:
            context: The context to retrieve required layers from
            layer_name: the name of the starting layer
            index: the index/order of the layer

        Returns:
            An iterable containing the levels and layer objects for all dependent layers
        """
        layer = context.layers[layer_name]
        yield index, layer
        try:
            for depends in layer.dependencies:
                for j, dep in cls.get_depends(context, depends, index + 1):
                    yield j, context.layers[dep.name]
        except AttributeError:
            # FileLayer won't have dependencies
            pass

    @classmethod
    def get_kernel_module(
        cls,
        context: interfaces.context.ContextInterface,
        layer_name: str,
        symbol_table: str,
    ):
        """Returns the kernel module based on the layer and symbol_table"""
        virtual_layer = context.layers[layer_name]
        if not isinstance(virtual_layer, layers.intel.Intel):
            raise TypeError("Virtual Layer is not an intel layer")

        kvo = virtual_layer.config["kernel_virtual_offset"]

        ntkrnlmp = context.module(symbol_table, layer_name=layer_name, offset=kvo)
        return ntkrnlmp

    @classmethod
    def get_kdbg_structure(
        cls,
        context: interfaces.context.ContextInterface,
        config_path: str,
        layer_name: str,
        symbol_table: str,
    ) -> interfaces.objects.ObjectInterface:
        """Returns the KDDEBUGGER_DATA64 structure for a kernel"""
        ntkrnlmp = cls.get_kernel_module(context, layer_name, symbol_table)

        native_types = context.symbol_space[symbol_table].natives

        kdbg_offset = ntkrnlmp.get_symbol("KdDebuggerDataBlock").address

        kdbg_table_name = intermed.IntermediateSymbolTable.create(
            context,
            interfaces.configuration.path_join(config_path, "kdbg"),
            "windows",
            "kdbg",
            native_types=native_types,
            class_types=extensions.kdbg.class_types,
        )

        kdbg = context.object(
            kdbg_table_name + constants.BANG + "_KDDEBUGGER_DATA64",
            offset=ntkrnlmp.offset + kdbg_offset,
            layer_name=layer_name,
        )

        return kdbg

    @classmethod
    def get_kuser_structure(
        cls,
        context: interfaces.context.ContextInterface,
        layer_name: str,
        symbol_table: str,
    ) -> interfaces.objects.ObjectInterface:
        """Returns the _KUSER_SHARED_DATA structure for a kernel"""
        virtual_layer = context.layers[layer_name]
        if not isinstance(virtual_layer, layers.intel.Intel):
            raise TypeError("Virtual Layer is not an intel layer")

        ntkrnlmp = cls.get_kernel_module(context, layer_name, symbol_table)

        # this is a hard-coded address in the Windows OS
        if virtual_layer.bits_per_register == 32:
            kuser_addr = 0xFFDF0000
        else:
            kuser_addr = 0xFFFFF78000000000

        kuser = ntkrnlmp.object(
            object_type="_KUSER_SHARED_DATA",
            layer_name=layer_name,
            offset=kuser_addr,
            absolute=True,
        )

        return kuser

    @classmethod
    def get_version_structure(
        cls,
        context: interfaces.context.ContextInterface,
        layer_name: str,
        symbol_table: str,
    ) -> interfaces.objects.ObjectInterface:
        """Returns the KdVersionBlock information from a kernel"""
        ntkrnlmp = cls.get_kernel_module(context, layer_name, symbol_table)

        vers_offset = ntkrnlmp.get_symbol("KdVersionBlock").address

        vers = ntkrnlmp.object(
            object_type="_DBGKD_GET_VERSION64",
            layer_name=layer_name,
            offset=vers_offset,
        )

        return vers

    @classmethod
    def get_ntheader_structure(
        cls,
        context: interfaces.context.ContextInterface,
        config_path: str,
        layer_name: str,
    ) -> interfaces.objects.ObjectInterface:
        """Gets the ntheader structure for the kernel of the specified layer"""
        virtual_layer = context.layers[layer_name]
        if not isinstance(virtual_layer, layers.intel.Intel):
            raise TypeError("Virtual Layer is not an intel layer")

        kvo = virtual_layer.config["kernel_virtual_offset"]

        pe_table_name = intermed.IntermediateSymbolTable.create(
            context,
            interfaces.configuration.path_join(config_path, "pe"),
            "windows",
            "pe",
            class_types=extensions.pe.class_types,
        )

        dos_header = context.object(
            pe_table_name + constants.BANG + "_IMAGE_DOS_HEADER",
            offset=kvo,
            layer_name=layer_name,
        )

        nt_header = dos_header.get_nt_header()

        return nt_header

    def _generator(self):
        kernel = self.context.modules[self.config["kernel"]]

        layer_name = kernel.layer_name
        symbol_table = kernel.symbol_table_name
        layer = self.context.layers[layer_name]
        table = self.context.symbol_space[symbol_table]

        kdbg = self.get_kdbg_structure(
            self.context, self.config_path, layer_name, symbol_table
        )

        yield (0, ("Kernel Base", hex(layer.config["kernel_virtual_offset"])))
        yield (0, ("DTB", hex(layer.config["page_map_offset"])))
        yield (0, ("Symbols", table.config["isf_url"]))
        yield (
            0,
            ("Is64Bit", str(symbols.symbol_table_is_64bit(self.context, symbol_table))),
        )
        yield (
            0,
            ("IsPAE", str(self.context.layers[layer_name].metadata.get("pae", False))),
        )

        for i, layer in self.get_depends(self.context, layer_name):
            yield (0, (layer.name, f"{i} {layer.__class__.__name__}"))

        if kdbg.Header.OwnerTag == 0x4742444B:
            yield (0, ("KdDebuggerDataBlock", hex(kdbg.vol.offset)))
            yield (0, ("NTBuildLab", kdbg.get_build_lab()))
            yield (0, ("CSDVersion", str(kdbg.get_csdversion())))

        vers = self.get_version_structure(self.context, layer_name, symbol_table)

        yield (0, ("KdVersionBlock", hex(vers.vol.offset)))
        yield (0, ("Major/Minor", f"{vers.MajorVersion}.{vers.MinorVersion}"))
        yield (0, ("MachineType", str(vers.MachineType)))

        ntkrnlmp = self.get_kernel_module(self.context, layer_name, symbol_table)

        cpu_count_offset = ntkrnlmp.get_symbol("KeNumberProcessors").address

        cpu_count = ntkrnlmp.object(
            object_type="unsigned int", layer_name=layer_name, offset=cpu_count_offset
        )

        yield (0, ("KeNumberProcessors", str(cpu_count)))

        kuser = self.get_kuser_structure(self.context, layer_name, symbol_table)

        yield (0, ("SystemTime", str(kuser.SystemTime.get_time())))
        yield (
            0,
            (
                "NtSystemRoot",
                str(
                    kuser.NtSystemRoot.cast(
                        "string", encoding="utf-16", errors="replace", max_length=260
                    )
                ),
            ),
        )
        yield (0, ("NtProductType", str(kuser.NtProductType.description)))
        yield (0, ("NtMajorVersion", str(kuser.NtMajorVersion)))
        yield (0, ("NtMinorVersion", str(kuser.NtMinorVersion)))
        # yield (0, ("KdDebuggerEnabled", "True" if kuser.KdDebuggerEnabled else "False"))
        # yield (0, ("SafeBootMode", "True" if kuser.SafeBootMode else "False"))

        nt_header = self.get_ntheader_structure(
            self.context, self.config_path, layer_name
        )

        yield (
            0,
            (
                "PE MajorOperatingSystemVersion",
                str(nt_header.OptionalHeader.MajorOperatingSystemVersion),
            ),
        )
        yield (
            0,
            (
                "PE MinorOperatingSystemVersion",
                str(nt_header.OptionalHeader.MinorOperatingSystemVersion),
            ),
        )

        yield (0, ("PE Machine", str(nt_header.FileHeader.Machine)))
        yield (
            0,
            (
                "PE TimeDateStamp",
                time.asctime(time.gmtime(nt_header.FileHeader.TimeDateStamp)),
            ),
        )

    def run(self):
        return TreeGrid([("Variable", str), ("Value", str)], self._generator())


# https://skyeong.net/186

import numpy as np
from sklearn.datasets import fetch_openml

mnist = fetch_openml("MNIST_784")

X = mnist.data / 255.0
y = mnist.target

print (X.shape, y.shape)

import pandas as pd

feat_cols = ['pixel'+str(i) for i in range(X.shape[1]) ]

df = pd.DataFrame(X,columns=feat_cols)
df['label'] = y
df['label'] = df['label'].apply(lambda i: str(i))

X, y = None, None

print( 'Size of the dataframe: {}'.format(df.shape) )

rndperm = np.random.permutation(df.shape[0])

import matplotlib.pyplot as plt

# Plot the graph
plt.gray()
fig = plt.figure( figsize=(16,7) )
for i in range(0,30):
    ax  = fig.add_subplot(3,10,i+1,title='Digit: ' + str(df.loc[rndperm[i], 'label']) )
    ax.matshow(df.loc[rndperm[i], feat_cols].values.reshape((28,28)).astype(float))

plt.show()

from sklearn.decomposition import PCA

pca = PCA(n_components=3)
pca_result = pca.fit_transform(df[feat_cols].values)

df['pca-one'] = pca_result[:,0]
df['pca-two'] = pca_result[:,1]
df['pca-three'] = pca_result[:,2]

print( 'Explained variation per principle component: {}'.format(pca.explained_variance_ratio_))