#!/usr/bin/env python
# -*- coding:utf-8 -*-
#@Time  : 2020/4/12 21:00
#@Author: Kevin.Liu
#@File  : linearRegression.py

from sklearn.linear_model import LinearRegression
from sklearn import datasets
from sklearn.model_selection import train_test_split
import matplotlib.pyplot as plt
import numpy as np

X = np.linspace(0, 10, 50).reshape(-1, 1)
# print(X)
y = np.random.randint(2, 8, size=1)*X
# print(y)
# print(y/X)

lr = LinearRegression()
lr.fit(X, y)
# coefficient 系数（斜率）个数：属性的个数决定
# w ---> weight 权重
# print(lr.coef_)

# 线性代数中矩阵运算
# print(np.linalg.inv(X.T.dot(X)).dot(X.T).dot(y))

# 波士顿房价
boston = datasets.load_boston()
X = boston['data']
y = boston['target']
# print(X.shape)
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2)
# lr = LinearRegression(fit_intercept=False)
# lr.fit(X_train, y_train)
# w = lr.coef_
# # print(lr.coef_, lr.intercept_)
# # 自己计算预测的结果
# print(X_test.dot(w).round(2)[:25])
# # 算法预测的结果
# print(lr.predict(X_test).round(2)[:25])
# # 真实结果
# print(y_test[:25])

lr = LinearRegression(fit_intercept=True)
lr.fit(X_train, y_train)
w = lr.coef_
# print(lr.coef_, lr.intercept_)
# 自己计算预测的结果 --> 根据斜率和截距构造方程，进行求解的结果
print((X_test.dot(w) + lr.intercept_).round(2)[:25])
# 算法预测的结果
print(lr.predict(X_test).round(2)[:25])
# 真实结果
print(y_test[:25])


"""
A simple baseline for a question answering system.
"""
import os
from pathlib import Path
import yaml
import openai
import gradio as gr
from annoy import AnnoyIndex
from PIL import Image
from transformers import BlipProcessor, BlipForConditionalGeneration
from dotenv import load_dotenv
load_dotenv()
openai.api_key = os.environ['OPENAI_API_KEY']

# --- load pre-processed chunks --- #
with open(Path(__file__).resolve().parent / "openai27052023.yaml", 'r') as f:
    paper = yaml.safe_load(f)
sentences = paper['sentences']

# --- embed chunks --- #
print("embedding chunks...")
embeddings = [
    r['embedding']
    for r in openai.Embedding.create(input=sentences, model='text-embedding-ada-002')['data']
]

# --- index embeddings for efficient search (using Spotify's annoy)--- #
hidden_size = len(embeddings[0])
index = AnnoyIndex(hidden_size, 'angular')  #  "angular" =  cosine
for i, e in enumerate(embeddings):
    index.add_item(i , e)
index.build(10)  # build 10 trees for efficient search

# --- iteratively answer questions (retrieve & generate) --- #

processor = BlipProcessor.from_pretrained("Salesforce/blip-image-captioning-base")
model = BlipForConditionalGeneration.from_pretrained("Salesforce/blip-image-captioning-base")

def run_on_cpu(image:str=None):
    if image is None:
        image = 'https://storage.googleapis.com/sfr-vision-language-research/BLIP/demo.jpg'
    # conditional image captioning
    text = "a photography of"
    inputs = processor(image, text, return_tensors="pt")
    out = model.generate(**inputs)
    print(processor.decode(out[0], skip_special_tokens=True))
    # >>> a photography of a woman and her dog
    # unconditional image captioning
    inputs = processor(image, return_tensors="pt")
    out = model.generate(**inputs)
    caption = processor.decode(out[0], skip_special_tokens=True)
    print(caption)
    # a woman sitting on the beach with her dog
    return caption

def respond(query, image, chat_history):
    if image is not None:
        caption = run_on_cpu(image)
        prompt = f"""
        The following is the description of an image:
        {caption}
        """
    embedding =  openai.Embedding.create(input = [query], model='text-embedding-ada-002')['data'][0]['embedding']
    # get nearest neighbors by vectors
    indices, distances = index.get_nns_by_vector(embedding,
                                                n=3,  # return top 3
                                                include_distances=True)
    results =  [
        (sentences[i], d)
        for i, d in zip(indices, distances)
    ]
    # with this, generate an answer
    excerpts = [res[0] for res in results]
    excerpts = '\n'.join([f'[{i}]. {excerpt}' for i, excerpt in enumerate(excerpts, start=1)])
    prompt = prompt + f"""
    user query:
    {query}

    title of the paper:
    {paper['title']}

    excerpts:
    {excerpts}
    ---
    given the excerpts from the paper above, answer the user query.
    In your answer, make sure to cite the excerpts by its number wherever appropriate.
    Note, however, that the excerpts may not be relevant to the user query.
    """
    gpt_role = """
    You are a human who speaks Korean and interacts with users like a exaggerate comedian.
    Don't let users know you're an AI.
    Never give out your any personal information.
    """
    chat_completion = openai.ChatCompletion.create(model="gpt-3.5-turbo",
                                                    messages=[
                                                        {"role": "system", "content": gpt_role},
                                                        {"role": "user", "content": prompt}
                                                    ])
    answer = chat_completion.choices[0].message.content
    chat_history.append((query, answer))
    return "", None, chat_history

with gr.Blocks(css='style.css', theme=gr.themes.Soft()) as demo:
    DESCRIPTION = "# Chatbot"
    gr.Markdown(DESCRIPTION)
    with gr.Group(elem_id="gr-container"):
        chatbot = gr.Chatbot().style(height=768)
        msg = gr.Textbox()
        image = gr.Image().style(height=512)
        clear = gr.ClearButton([msg, chatbot, image])
        msg.submit(respond, [msg, image, chatbot], [msg, image, chatbot])

demo.launch()


# It is difficult to invoke the Python coverage tool externally with Jython, so
# it is being invoked internally here:
from __future__ import division
import net.grinder.script.Grinder

package_path = net.grinder.script.Grinder.grinder.getProperties().getProperty(
    "grinder.package_path")
import sys

sys.path.append(package_path)
from coverage import coverage

cov = coverage()
cov.start()

import time
import utils
import ingest
import ingestenum
import query
import annotationsingest
import unittest
import random
import math
import grinder

try:
    from com.xhaus.jyson import JysonCodec as json
except ImportError:
    import json
import pprint

pp = pprint.pprint
sleep_time = -1
get_url = None
post_url = None
post_payload = None

def mock_sleep(cls, x):
    global sleep_time
    sleep_time = x

class MockReq():
    def POST(self, url, payload):
        global post_url, post_payload
        post_url = url
        post_payload = payload
        return url, payload

    def GET(self, url):
        global get_url
        get_url = url
        return url

class BluefloodTests(unittest.TestCase):
    def setUp(self):
        self.real_shuffle = random.shuffle
        self.real_randint = random.randint
        self.real_time = utils.AbstractThread.time
        self.real_sleep = utils.AbstractThread.sleep
        self.tm = ingest.ThreadManager(net.grinder.script.Grinder.grinder)
        req = MockReq()
        ingest.IngestThread.request = req
        ingestenum.EnumIngestThread.request = req
        annotationsingest.AnnotationsIngestThread.request = req
        for x in query.QueryThread.query_types:
            x.query_request = req
        random.shuffle = lambda x: None
        random.randint = lambda x, y: 0
        utils.AbstractThread.time = lambda x: 1000
        utils.AbstractThread.sleep = mock_sleep

        test_config = {'report_interval': (1000 * 6),
                       'num_tenants': 3,
                       'enum_num_tenants': 4,
                       'annotations_num_tenants': 3,
                       'metrics_per_tenant': 7,
                       'enum_metrics_per_tenant': 2,
                       'annotations_per_tenant': 2,
                       'batch_size': 3,
                       'ingest_concurrency': 2,
                       'enum_ingest_concurrency': 2,
                       'query_concurrency': 20,
                       'annotations_concurrency': 2,
                       'singleplot_per_interval': 11,
                       'multiplot_per_interval': 10,
                       'search_queries_per_interval': 9,
                       'enum_search_queries_per_interval': 9,
                       'enum_single_plot_queries_per_interval': 10,
                       'enum_multiplot_per_interval': 10,
                       'annotations_queries_per_interval': 8,
                       'name_fmt': "int.abcdefg.hijklmnop.qrstuvw.xyz.ABCDEFG.HIJKLMNOP.QRSTUVW.XYZ.abcdefg.hijklmnop.qrstuvw.xyz.met.%d",
                       'num_nodes': 2}

        ingest.default_config.update(test_config)

    def test_init_process(self):
        # confirm that threadnum 0 is an ingest thread
        t1 = self.tm.setup_thread(0)
        self.assertEqual(type(t1), ingest.IngestThread)

        # confirm that the threadnum after all ingest threads is
        # EnumIngestThread
        t1 = self.tm.setup_thread(
            ingestenum.default_config['enum_ingest_concurrency'])
        self.assertEqual(type(t1), ingestenum.EnumIngestThread)

        # confirm that the threadnum after all ingest threads is a query thread
        t1 = self.tm.setup_thread(ingest.default_config['ingest_concurrency'] +
                                  ingestenum.default_config[
                                      'enum_ingest_concurrency'])
        self.assertEqual(type(t1), query.QueryThread)

        # confirm that the threadnum after all ingest+query threads is an
        # annotations query thread
        t1 = self.tm.setup_thread(ingest.default_config['ingest_concurrency'] +
                                  ingestenum.default_config[
                                      'enum_ingest_concurrency'] +
                                  ingest.default_config['query_concurrency'])
        self.assertEqual(type(t1), annotationsingest.AnnotationsIngestThread)

        # confirm that a threadnum after all valid thread types raises an
        # exception
        tot_threads = (
            ingest.default_config['ingest_concurrency'] +
            ingest.default_config['enum_ingest_concurrency'] +
            ingest.default_config['query_concurrency'] +
            ingest.default_config['annotations_concurrency'])
        self.assertRaises(Exception, self.tm.setup_thread, tot_threads)

        # confirm that the correct batches of ingest metrics are created for
        # worker 0
        self.tm.create_all_metrics(0)

        # confirm annotationsingest
        self.assertEqual(annotationsingest.AnnotationsIngestThread.annotations,
                         [[0, 0], [0, 1], [1, 0], [1, 1]])

        thread = annotationsingest.AnnotationsIngestThread(0)
        self.assertEqual(thread.slice, [[0, 0], [0, 1]])

        thread = annotationsingest.AnnotationsIngestThread(1)
        self.assertEqual(thread.slice, [[1, 0], [1, 1]])

        # confirm enum metrics ingest
        self.assertEqual(ingestenum.EnumIngestThread.metrics,
                         [
                             [[0, 0], [0, 1], [1, 0]],
                             [[1, 1]]
                         ])

        thread = ingestenum.EnumIngestThread(0)
        self.assertEqual(thread.slice, [[[0, 0], [0, 1], [1, 0]]])

        thread = ingestenum.EnumIngestThread(1)
        self.assertEqual(thread.slice, [[[1, 1]]])

        # confirm metrics ingest
        self.assertEqual(ingest.IngestThread.metrics,
                         [[[0, 0], [0, 1], [0, 2]],
                          [[0, 3], [0, 4], [0, 5]],
                          [[0, 6], [1, 0], [1, 1]],
                          [[1, 2], [1, 3], [1, 4]],
                          [[1, 5], [1, 6]]])

        # confirm that the correct batch slices are created for individual
        # threads
        thread = ingest.IngestThread(0)
        self.assertEqual(thread.slice,
                         [[[0, 0], [0, 1], [0, 2]],
                          [[0, 3], [0, 4], [0, 5]],
                          [[0, 6], [1, 0], [1, 1]]])
        thread = ingest.IngestThread(1)
        self.assertEqual(thread.slice,
                         [[[1, 2], [1, 3], [1, 4]],
                          [[1, 5], [1, 6]]])

        # confirm that the number of queries is correctly distributed across
        #  each thread in this worker process

        num_query_nodes = query.default_config['num_nodes']
        single_plot_queries_agent0 = int(math.ceil(
            query.default_config['singleplot_per_interval'] / num_query_nodes))
        multi_plot_queries_agent0 = int(math.ceil(
            query.default_config['multiplot_per_interval'] / num_query_nodes))
        search_queries_agent0 = int(math.ceil(
            query.default_config[
                'search_queries_per_interval'] / num_query_nodes))
        enum_search_queries_agent0 = int(math.ceil(
            query.default_config[
                'enum_search_queries_per_interval'] / num_query_nodes))
        enum_single_plot_queries_agent0 = int(math.ceil(
            query.default_config[
                'enum_single_plot_queries_per_interval'] / num_query_nodes))
        enum_multi_plot_queries_agent0 = int(math.ceil(
            query.default_config[
                'enum_multiplot_per_interval'] / num_query_nodes))
        annotation_queries_agent0 = int(math.ceil(
            query.default_config[
                'annotations_queries_per_interval'] / num_query_nodes))

        self.assertEqual(
            query.QueryThread.queries,
            ([query.SinglePlotQuery] * single_plot_queries_agent0 +
             [query.MultiPlotQuery] * multi_plot_queries_agent0 +
             [query.SearchQuery] * search_queries_agent0 +
             [query.EnumSearchQuery] * enum_search_queries_agent0 +
             [query.EnumSinglePlotQuery] * enum_single_plot_queries_agent0 +
             [query.AnnotationsQuery] * annotation_queries_agent0) +
            [query.EnumMultiPlotQuery] * enum_multi_plot_queries_agent0)

        thread = query.QueryThread(0)
        self.assertEqual(thread.slice, [query.SinglePlotQuery] * 2)

        thread = query.QueryThread(3)
        self.assertEqual(thread.slice, [query.MultiPlotQuery] * 2)

        thread = query.QueryThread(6)
        self.assertEqual(thread.slice, [query.SearchQuery] * 2)

        thread = query.QueryThread(9)
        self.assertEqual(thread.slice, [query.EnumSearchQuery] * 2)

        thread = query.QueryThread(12)
        self.assertEqual(thread.slice, [query.EnumSinglePlotQuery] * 2)

        thread = query.QueryThread(14)
        self.assertEqual(thread.slice, [query.AnnotationsQuery] * 2)

        thread = query.QueryThread(16)
        self.assertEqual(thread.slice, [query.EnumMultiPlotQuery] * 1)

        # confirm that the correct batches of ingest metrics are created for
        # worker 1
        self.tm.create_all_metrics(1)
        self.assertEqual(ingest.IngestThread.metrics,
                         [[[2, 0], [2, 1], [2, 2]],
                          [[2, 3], [2, 4], [2, 5]],
                          [[2, 6]]])

        self.assertEqual(annotationsingest.AnnotationsIngestThread.annotations,
                         [[2, 0], [2, 1]])

        thread = ingest.IngestThread(0)
        self.assertEqual(thread.slice,
                         [[[2, 0], [2, 1], [2, 2]],
                          [[2, 3], [2, 4], [2, 5]]])
        thread = ingest.IngestThread(1)
        self.assertEqual(thread.slice,
                         [[[2, 6]]])

        # confirm that the correct batches of queries are created for worker 1
        single_plot_queries_agent1 = \
            query.default_config['singleplot_per_interval'] - \
            single_plot_queries_agent0
        multi_plot_queries_agent1 = \
            query.default_config['multiplot_per_interval'] - \
            multi_plot_queries_agent0
        search_queries_agent1 = \
            query.default_config['search_queries_per_interval'] - \
            search_queries_agent0
        enum_search_queries_agent1 = \
            query.default_config['enum_search_queries_per_interval'] - \
            enum_search_queries_agent0
        enum_single_plot_queries_agent1 = \
            query.default_config['enum_single_plot_queries_per_interval'] - \
            enum_single_plot_queries_agent0
        annotation_queries_agent1 = \
            query.default_config['annotations_queries_per_interval'] - \
            annotation_queries_agent0
        enum_multi_plot_queries_agent1 = \
            query.default_config['enum_multiplot_per_interval'] - \
            enum_multi_plot_queries_agent0

        self.assertEqual(
            query.QueryThread.queries,
            ([query.SinglePlotQuery] * single_plot_queries_agent1 +
             [query.MultiPlotQuery] * multi_plot_queries_agent1 +
             [query.SearchQuery] * search_queries_agent1 +
             [query.EnumSearchQuery] * enum_search_queries_agent1 +
             [query.EnumSinglePlotQuery] * enum_single_plot_queries_agent1 +
             [query.AnnotationsQuery] * annotation_queries_agent1) +
            [query.EnumMultiPlotQuery] * enum_multi_plot_queries_agent1)

        thread = query.QueryThread(0)
        self.assertEqual(thread.slice, [query.SinglePlotQuery] * 2)

        thread = query.QueryThread(4)
        self.assertEqual(thread.slice, [query.MultiPlotQuery] * 2)

        thread = query.QueryThread(6)
        self.assertEqual(thread.slice, [query.SearchQuery] * 2)

        thread = query.QueryThread(8)
        self.assertEqual(thread.slice, [query.EnumSearchQuery] * 2)

        thread = query.QueryThread(10)
        self.assertEqual(thread.slice, [query.EnumSinglePlotQuery] * 2)

        thread = query.QueryThread(12)
        self.assertEqual(thread.slice, [query.AnnotationsQuery] * 1)

        thread = query.QueryThread(16)
        self.assertEqual(thread.slice, [query.EnumMultiPlotQuery] * 1)

    def test_generate_payload(self):
        self.tm.create_all_metrics(1)
        thread = ingest.IngestThread(0)
        payload = json.loads(
            thread.generate_payload(0, [[2, 3], [2, 4], [2, 5]]))
        valid_payload = [{u'collectionTime': 0,
                          u'metricName': u'int.abcdefg.hijklmnop.qrstuvw.xyz.ABCDEFG.HIJKLMNOP.QRSTUVW.XYZ.abcdefg.hijklmnop.qrstuvw.xyz.met.3',
                          u'metricValue': 0,
                          u'tenantId': u'2',
                          u'ttlInSeconds': 172800,
                          u'unit': u'days'},
                         {u'collectionTime': 0,
                          u'metricName': u'int.abcdefg.hijklmnop.qrstuvw.xyz.ABCDEFG.HIJKLMNOP.QRSTUVW.XYZ.abcdefg.hijklmnop.qrstuvw.xyz.met.4',
                          u'metricValue': 0,
                          u'tenantId': u'2',
                          u'ttlInSeconds': 172800,
                          u'unit': u'days'},
                         {u'collectionTime': 0,
                          u'metricName': u'int.abcdefg.hijklmnop.qrstuvw.xyz.ABCDEFG.HIJKLMNOP.QRSTUVW.XYZ.abcdefg.hijklmnop.qrstuvw.xyz.met.5',
                          u'metricValue': 0,
                          u'tenantId': u'2',
                          u'ttlInSeconds': 172800,
                          u'unit': u'days'}]
        self.assertEqual(payload, valid_payload)

    def test_generate_enum_payload(self):
        self.tm.create_all_metrics(1)
        thread = ingestenum.EnumIngestThread(0)
        payload = json.loads(thread.generate_payload(1, [[2, 1], [2, 2]]))
        valid_payload = [{u'timestamp': 1,
                          u'tenantId': u'2',
                          u'enums': [{u'value': u'e_g_1_0',
                                      u'name': utils.generate_enum_metric_name(
                                          1)}]},
                         {u'timestamp': 1,
                          u'tenantId': u'2',
                          u'enums': [{u'value': u'e_g_2_0',
                                      u'name': utils.generate_enum_metric_name(
                                          2)}]}
                         ]
        self.assertEqual(payload, valid_payload)

    def test_generate_annotations_payload(self):
        self.tm.create_all_metrics(1)
        thread = annotationsingest.AnnotationsIngestThread(0)
        payload = json.loads(thread.generate_payload(0, 3))
        valid_payload = {
            'what': 'annotation int.abcdefg.hijklmnop.qrstuvw.xyz.ABCDEFG.HIJKLMNOP.QRSTUVW.XYZ.abcdefg.hijklmnop.qrstuvw.xyz.met.3',
            'when': 0,
            'tags': 'tag',
            'data': 'data'}
        self.assertEqual(payload, valid_payload)

    def test_annotationsingest_make_request(self):
        global sleep_time
        thread = annotationsingest.AnnotationsIngestThread(0)
        thread.slice = [[2, 0]]
        thread.position = 0
        thread.finish_time = 10000
        valid_payload = {
            "what": "annotation int.abcdefg.hijklmnop.qrstuvw.xyz.ABCDEFG.HIJKLMNOP.QRSTUVW.XYZ.abcdefg.hijklmnop.qrstuvw.xyz.met.0",
            "when": 1000, "tags": "tag", "data": "data"}

        url, payload = thread.make_request(pp)
        # confirm request generates proper URL and payload
        self.assertEqual(
            url,
            'http://qe01.metrics-ingest.api.rackspacecloud.com/v2.0/2/events')
        self.assertEqual(eval(payload), valid_payload)

        # confirm request increments position if not at end of report interval
        self.assertEqual(thread.position, 1)
        self.assertEqual(thread.finish_time, 10000)
        thread.position = 2
        thread.make_request(pp)

        # confirm request resets position at end of report interval
        self.assertEqual(sleep_time, 9000)
        self.assertEqual(thread.position, 1)
        self.assertEqual(thread.finish_time, 16000)

    def test_ingest_make_request(self):
        global sleep_time
        thread = ingest.IngestThread(0)
        thread.slice = [[[2, 0], [2, 1]]]
        thread.position = 0
        thread.finish_time = 10000
        valid_payload = [
            {"collectionTime": 1000, "ttlInSeconds": 172800, "tenantId": "2",
             "metricValue": 0, "unit": "days",
             "metricName": "int.abcdefg.hijklmnop.qrstuvw.xyz.ABCDEFG.HIJKLMNOP.QRSTUVW.XYZ.abcdefg.hijklmnop.qrstuvw.xyz.met.0"},
            {"collectionTime": 1000, "ttlInSeconds": 172800, "tenantId": "2",
             "metricValue": 0, "unit": "days",
             "metricName": "int.abcdefg.hijklmnop.qrstuvw.xyz.ABCDEFG.HIJKLMNOP.QRSTUVW.XYZ.abcdefg.hijklmnop.qrstuvw.xyz.met.1"}]

        url, payload = thread.make_request(pp)
        # confirm request generates proper URL and payload
        self.assertEqual(url,
                         'http://qe01.metrics-ingest.api.rackspacecloud.com/v2.0/tenantId/ingest/multi')
        self.assertEqual(eval(payload), valid_payload)

        # confirm request increments position if not at end of report interval
        self.assertEqual(thread.position, 1)
        self.assertEqual(thread.finish_time, 10000)
        thread.position = 2
        thread.make_request(pp)
        # confirm request resets position at end of report interval
        self.assertEqual(sleep_time, 9000)
        self.assertEqual(thread.position, 1)
        self.assertEqual(thread.finish_time, 16000)

    def test_ingest_enum_make_request(self):
        global sleep_time
        thread = ingestenum.EnumIngestThread(0)
        thread.slice = [[[2, 0], [2, 1]]]
        thread.position = 0
        thread.finish_time = 10000
        valid_payload = [{'tenantId': '2', 'timestamp': 1000, 'enums': [
            {'value': 'e_g_0_0', 'name': utils.generate_enum_metric_name(0)}]},
                         {'tenantId': '2', 'timestamp': 1000, 'enums': [
                             {'value': 'e_g_1_0',
                              'name': utils.generate_enum_metric_name(1)}]}]

        url, payload = thread.make_request(pp)
        # confirm request generates proper URL and payload
        self.assertEqual(url,
                         'http://qe01.metrics-ingest.api.rackspacecloud.com/v2.0/tenantId/ingest/aggregated/multi')
        self.assertEqual(eval(payload), valid_payload)

        # confirm request increments position if not at end of report interval
        self.assertEqual(thread.position, 1)
        self.assertEqual(thread.finish_time, 10000)
        thread.position = 2
        thread.make_request(pp)
        # confirm request resets position at end of report interval
        self.assertEqual(sleep_time, 9000)
        self.assertEqual(thread.position, 1)
        self.assertEqual(thread.finish_time, 16000)

    def test_query_make_request(self):
        thread = query.QueryThread(0)
        thread.slice = [query.SinglePlotQuery, query.SearchQuery,
                        query.MultiPlotQuery, query.AnnotationsQuery,
                        query.EnumSearchQuery, query.EnumSinglePlotQuery,
                        query.EnumMultiPlotQuery]
        thread.position = 0
        thread.make_request(pp)
        self.assertEqual(get_url,
                         "http://qe01.metrics.api.rackspacecloud.com/v2.0/0/views/int.abcdefg.hijklmnop.qrstuvw.xyz.ABCDEFG.HIJKLMNOP.QRSTUVW.XYZ.abcdefg.hijklmnop.qrstuvw.xyz.met.0?from=-86399000&to=1000&resolution=FULL")

        random.randint = lambda x, y: 10
        thread.make_request(pp)
        self.assertEqual(get_url,
                         "http://qe01.metrics.api.rackspacecloud.com/v2.0/10/metrics/search?query=int.abcdefg.hijklmnop.qrstuvw.xyz.ABCDEFG.HIJKLMNOP.QRSTUVW.XYZ.abcdefg.hijklmnop.qrstuvw.xyz.met.*")

        random.randint = lambda x, y: 20
        thread.make_request(pp)
        self.assertEqual(post_url,
                         "http://qe01.metrics.api.rackspacecloud.com/v2.0/20/views?from=-86399000&to=1000&resolution=FULL")
        self.assertEqual(eval(post_payload), [
            "int.abcdefg.hijklmnop.qrstuvw.xyz.ABCDEFG.HIJKLMNOP.QRSTUVW.XYZ.abcdefg.hijklmnop.qrstuvw.xyz.met.0",
            "int.abcdefg.hijklmnop.qrstuvw.xyz.ABCDEFG.HIJKLMNOP.QRSTUVW.XYZ.abcdefg.hijklmnop.qrstuvw.xyz.met.1",
            "int.abcdefg.hijklmnop.qrstuvw.xyz.ABCDEFG.HIJKLMNOP.QRSTUVW.XYZ.abcdefg.hijklmnop.qrstuvw.xyz.met.2",
            "int.abcdefg.hijklmnop.qrstuvw.xyz.ABCDEFG.HIJKLMNOP.QRSTUVW.XYZ.abcdefg.hijklmnop.qrstuvw.xyz.met.3",
            "int.abcdefg.hijklmnop.qrstuvw.xyz.ABCDEFG.HIJKLMNOP.QRSTUVW.XYZ.abcdefg.hijklmnop.qrstuvw.xyz.met.4",
            "int.abcdefg.hijklmnop.qrstuvw.xyz.ABCDEFG.HIJKLMNOP.QRSTUVW.XYZ.abcdefg.hijklmnop.qrstuvw.xyz.met.5",
            "int.abcdefg.hijklmnop.qrstuvw.xyz.ABCDEFG.HIJKLMNOP.QRSTUVW.XYZ.abcdefg.hijklmnop.qrstuvw.xyz.met.6",
            "int.abcdefg.hijklmnop.qrstuvw.xyz.ABCDEFG.HIJKLMNOP.QRSTUVW.XYZ.abcdefg.hijklmnop.qrstuvw.xyz.met.7",
            "int.abcdefg.hijklmnop.qrstuvw.xyz.ABCDEFG.HIJKLMNOP.QRSTUVW.XYZ.abcdefg.hijklmnop.qrstuvw.xyz.met.8",
            "int.abcdefg.hijklmnop.qrstuvw.xyz.ABCDEFG.HIJKLMNOP.QRSTUVW.XYZ.abcdefg.hijklmnop.qrstuvw.xyz.met.9"])

        random.randint = lambda x, y: 30
        thread.make_request(pp)
        self.assertEqual(get_url,
                         "http://qe01.metrics.api.rackspacecloud.com/v2.0/30/events/getEvents?from=-86399000&until=1000")

        random.randint = lambda x, y: 40
        thread.make_request(pp)
        self.assertEqual(get_url,
                         "http://qe01.metrics.api.rackspacecloud.com/v2.0/40/metrics/search?query=enum_grinder_int.abcdefg.hijklmnop.qrstuvw.xyz.ABCDEFG.HIJKLMNOP.QRSTUVW.XYZ.abcdefg.hijklmnop.qrstuvw.xyz.met.*&include_enum_values=true")

        random.randint = lambda x, y: 50
        thread.make_request(pp)
        self.assertEqual(get_url,
                         "http://qe01.metrics.api.rackspacecloud.com/v2.0/50/views/enum_grinder_int.abcdefg.hijklmnop.qrstuvw.xyz.ABCDEFG.HIJKLMNOP.QRSTUVW.XYZ.abcdefg.hijklmnop.qrstuvw.xyz.met.50?from=-86399000&to=1000&resolution=FULL")

        random.randint = lambda x, y: 4
        thread.make_request(pp)
        self.assertEqual(post_url,
                         "http://qe01.metrics.api.rackspacecloud.com/v2.0/4/views?from=-86399000&to=1000&resolution=FULL")
        self.assertEqual(eval(post_payload), [
            "enum_grinder_int.abcdefg.hijklmnop.qrstuvw.xyz.ABCDEFG.HIJKLMNOP.QRSTUVW.XYZ.abcdefg.hijklmnop.qrstuvw.xyz.met.0",
            "enum_grinder_int.abcdefg.hijklmnop.qrstuvw.xyz.ABCDEFG.HIJKLMNOP.QRSTUVW.XYZ.abcdefg.hijklmnop.qrstuvw.xyz.met.1",
            "enum_grinder_int.abcdefg.hijklmnop.qrstuvw.xyz.ABCDEFG.HIJKLMNOP.QRSTUVW.XYZ.abcdefg.hijklmnop.qrstuvw.xyz.met.2",
            "enum_grinder_int.abcdefg.hijklmnop.qrstuvw.xyz.ABCDEFG.HIJKLMNOP.QRSTUVW.XYZ.abcdefg.hijklmnop.qrstuvw.xyz.met.3"])

    def tearDown(self):
        random.shuffle = self.real_shuffle
        random.randint = self.real_randint
        utils.AbstractThread.time = self.real_time
        utils.AbstractThread.sleep = self.real_sleep

# if __name__ == '__main__':
unittest.TextTestRunner().run(
    unittest.TestLoader().loadTestsFromTestCase(BluefloodTests))

cov.stop()
cov.save()

class TestRunner:
    def __init__(self):
        pass

    def __call__(self):
        pass


import matplotlib.pyplot as plt

# position visualization
def plot(new, ticker1, ticker2):
    """Visualise a position given a _signals_ object and two ticker names"""

    fig = plt.figure(figsize=(10, 5))
    bx = fig.add_subplot(111)
    bx2 = bx.twinx()

    # plot two different assets
    (l1,) = bx.plot(new.index, new["asset1"], c="#4abdac")
    (l2,) = bx2.plot(new.index, new["asset2"], c="#907163")

    (u1,) = bx.plot(
        new.loc[new["positions1"] == 1].index,
        new["asset1"][new["positions1"] == 1],
        lw=0,
        marker="^",
        markersize=8,
        c="g",
        alpha=0.7,
    )
    (d1,) = bx.plot(
        new.loc[new["positions1"] == -1].index,
        new["asset1"][new["positions1"] == -1],
        lw=0,
        marker="v",
        markersize=8,
        c="r",
        alpha=0.7,
    )
    (u2,) = bx2.plot(
        new.loc[new["positions2"] == 1].index,
        new["asset2"][new["positions2"] == 1],
        lw=0,
        marker=2,
        markersize=9,
        c="g",
        alpha=0.9,
        markeredgewidth=3,
    )
    (d2,) = bx2.plot(
        new.loc[new["positions2"] == -1].index,
        new["asset2"][new["positions2"] == -1],
        lw=0,
        marker=3,
        markersize=9,
        c="r",
        alpha=0.9,
        markeredgewidth=3,
    )

    bx.set_ylabel(
        ticker1,
    )
    bx2.set_ylabel(ticker2, rotation=270)
    bx.yaxis.labelpad = 15
    bx2.yaxis.labelpad = 15
    bx.set_xlabel("Date")
    bx.xaxis.labelpad = 15

    plt.legend(
        [l1, l2, u1, d1, u2, d2],
        [
            ticker1,
            ticker2,
            "LONG {}".format(ticker1),
            "SHORT {}".format(ticker1),
            "LONG {}".format(ticker2),
            "SHORT {}".format(ticker2),
        ],
        loc=8,
    )

    plt.title("Pair Trading")
    plt.xlabel("Date")
    plt.grid(True)
    plt.show()


import os, sys
#os.add_dll_directory("C:/Program Files/NVIDIA GPU Computing Toolkit/CUDA/v11.2/bin")
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
from keras.layers import Input, Dense, Conv2D, MaxPooling2D, UpSampling2D, Lambda, Add, Reshape
from keras.models import load_model,Model
import tensorflow as tf
from keras.callbacks import ModelCheckpoint,EarlyStopping
from sklearn.model_selection import train_test_split
import pickle
from tqdm import tqdm as tqdm
from scipy.io import FortranFile
from scipy.interpolate import interp2d

from matplotlib import cm
#from colorspacious import cspace_converter
from collections import OrderedDict

#This part of the code creates the images of the decomposed fields
#with the already trained model

filenm='cnn2.hdf5' # File name of this model

seqs= 2
ix = 193
jx = 129
kx = 0
ivar = 2
ntime = 512
ntime1= 512
ntime2 = ntime1 + ntime*(seqs-2)
shape = (ix+1,jx+1,1,ivar,ntime2)

print('Number of snapshots =',ntime2)

var = np.zeros(shape)

print('Reading from prim files')
n = 2

while (n<=seqs):
    fname = 'prim_'
    ext = '.s8'
    num = str(n)
    if len(num)==1:
        fname = fname + '0'+ num + ext
    else:
        fname = fname + num + ext
    print('Opening ',fname)
    f = FortranFile(fname,'r')
    if n==2:
        b = f.read_reals(np.float).reshape(ix+1,jx+1,kx+1,
                                           ivar,ntime1,order = 'F')
        var[:,:,:,:,0:ntime1] = b
        last = ntime1
    else:
        b = f.read_reals(np.float).reshape(ix+1,jx+1,kx+1,
                                           ivar,ntime,order = 'F')
        var[:,:,:,:,last:ntime+last] = b
        last = ntime + last
        print(last)

    n = n+1
var = np.squeeze(var)
var = np.moveaxis(var,3,0)
#var = var[:,0:ix,0:jx,:]
print(var.shape)

model = load_model(filenm)
layer_outputs = [layer.output for layer in model.layers[:]]
print(len(layer_outputs))
activation_model = Model(inputs = model.input, outputs = layer_outputs)
print(var[0:1,:,:,:].shape)
activation = activation_model.predict(var[0:1,:,:,:])
#x = model.predict(var)
mode1 = activation[45]
mode2 = activation[46]
reconstructed = model.predict(var[0:1,:,:,:])#activation[47]
print(mode1.shape)
print(mode2.shape)
f = FortranFile('geo.s8','r')
grid = f.read_reals(np.float).reshape(ix+1,jx+1,kx+1,
                                   2,order = 'F')
grid = np.squeeze(grid)

m = int(jx+1)
l2norm = np.zeros(reconstructed.shape)
l2norm = np.sqrt(np.square(var[0:1,:,:,:] - reconstructed))
# inter = interp2d(grid[:,0:m,0],grid[:,0:m,1],reconstructed[0,:,0:m,0])
# reconstructed = inter(grid[:,0:m,0],grid[:,0:m,1])
#cmaps['Diverging'] = ['seismic']

#To reverse colormap
hot1 = plt.cm.get_cmap('hot')
hot = hot1.reversed()
plt.pcolormesh(grid[:,0:m,0],grid[:,0:m,1],reconstructed[0,:,0:m,0],cmap='jet',shading='gouraud')
#plt.margins(x=-.3,y=-0.35)
plt.axis('scaled')
plt.axis([-1,8,-3.5,3.5])
plt.ylabel('y',fontname = 'Times New Roman', fontsize = 20)
plt.xlabel('x',fontname = 'Times New Roman', fontsize = 20)
plt.xticks(fontname = 'Times New Roman', fontsize = 20)
plt.yticks(fontname = 'Times New Roman', fontsize = 20)
plt.tight_layout()
#plt.show()
plt.savefig('predicted_u.png')
plt.pcolormesh(grid[:,0:m,0],grid[:,0:m,1],reconstructed[0,:,0:m,1],cmap='jet',shading='gouraud')
plt.savefig('predicted_v.png')

plt.ylabel('y',fontname = 'Times New Roman', fontsize = 20)
plt.pcolormesh(grid[:,0:m,0],grid[:,0:m,1],var[0,:,0:m,0],cmap='jet',shading='gouraud')
plt.xlabel('x',fontname = 'Times New Roman', fontsize = 20)

#plt.imshow(var[0,:,0:m,0], interpolation = 'bilinear')
#plt.show()
plt.savefig('u.png')
plt.pcolormesh(grid[:,0:m,0],grid[:,0:m,1],var[0,:,0:m,1],cmap='jet',shading='gouraud')
plt.savefig('v.png')

plt.pcolormesh(grid[:,0:m,0],grid[:,0:m,1],l2norm[0,:,0:m,0],cmap=hot,shading='gouraud',vmin=0,vmax=0.1)

cb = plt.colorbar(shrink=0.75)
for l in cb.ax.yaxis.get_ticklabels():
    l.set_family("Times New Roman")
    l.set_size(14)
#cb.ticks(fontname = 'Times New Roman', fontsize = 15)
title = plt.title(r'$L_2$ norm error',fontname = 'Times New Roman', fontsize = 15)
plt.savefig('l2norm_u.png')
plt.pcolormesh(grid[:,0:m,0],grid[:,0:m,1],l2norm[0,:,0:m,1],cmap=hot,shading='gouraud',vmin=0,vmax=0.1)
plt.savefig('l2norm_v.png')
plt.pcolormesh(grid[:,0:m,0],grid[:,0:m,1],mode1[0,:,0:m,0],cmap='jet',shading='gouraud')
cb.remove()
title.remove()
#plt.ylabel('y',fontname = 'Times New Roman')
#plt.xlabel('x',fontname = 'Times New Roman')
#plt.show()
plt.savefig('mode1_u.png')
plt.pcolormesh(grid[:,0:m,0],grid[:,0:m,1],mode2[0,:,0:m,0],cmap='jet',shading='gouraud')
#plt.ylabel('y',fontname = 'Times New Roman')
#plt.xlabel('x',fontname = 'Times New Roman')
#plt.show()
plt.savefig('mode2_u.png')
plt.pcolormesh(grid[:,0:m,0],grid[:,0:m,1],mode1[0,:,0:m,1],cmap='jet',shading='gouraud')
#plt.ylabel('y',fontname = 'Times New Roman')
#plt.xlabel('x',fontname = 'Times New Roman')
#plt.show()
plt.savefig('mode1_v.png')
plt.pcolormesh(grid[:,0:m,0],grid[:,0:m,1],mode2[0,:,0:m,1],cmap='jet',shading='gouraud')
#plt.ylabel('y',fontname = 'Times New Roman')
#plt.xlabel('x',fontname = 'Times New Roman')
#plt.show()
plt.savefig('mode2_v.png')

# g = open('tec.plt')
# print('title="grid"',file=g)
# print('variables="x-grid" "y-grid" "u" "v"',file=g)
# print('zone t="grid" f=point, i=',ix+2,', j=',jx+2,file=g)
# j=0
# i=0
#   while  j<=jx+1:
#         while i<=ix+1:
#             print(geo(i,j,1),geo(i,j,2),x(0,i,j,1), var(i,j,0,2,400)


import numpy as np
import argparse
import os

from PIL import Image

def __get_img_to_raw_np(img_filepath):
    '''
    获取图像的原始数组

    参数：
        img_filepath (src): 图像文件的路径

    返回:
        numpy.ndarray: 图像的原始数组

    Raises:
        RuntimeError: 如果图像的通道数不等于3 (不是RGB图像)，将引发运行时错误。
    '''
    img_filepath = os.path.abspath(img_filepath)
    img = Image.open(img_filepath)
    img_ndarray = np.array(img)
    # 检查是否为 RGB 图像, shape[0]: h, shape[1] w, shape[2]: c
    if img_ndarray.shape[2]!=3:
        raise RuntimeError('Require image with rgb but channel is %d' % img_ndarray.shape[2])
    return img_ndarray

def __create_raw_from_jpg(img_filepath, dst, req_bgr_raw, save_uint8):
    '''
    从 JPEG 图像创建 SNPE 所需的原始数组, 最后保存为 .raw 文件

    Args:
        img_filepath (str): JPEG图像文件的路径。
        req_bgr_raw (bool): 是否需要将图像数据转换为BGR格式。
        save_uint8 (bool): 是否将图像数据保存为8位整数类型(uint8)。

    '''
    img_raw = __get_img_to_raw_np(img_filepath)

    snpe_raw = img_raw.astype(np.float32)

    if req_bgr_raw:
        snpe_raw = snpe_raw[..., ::-1]

    if save_uint8:
        snpe_raw = snpe_raw.astype(np.uint8)
    else:
        snpe_raw = snpe_raw.astype(np.float32)

    filename, _ = os.path.splitext(os.path.basename(img_filepath))
    snpe_raw_filename = os.path.join(dst, filename + '.raw')
    snpe_raw.tofile(snpe_raw_filename)

def convert_images_to_raw(src, dst, req_bgr_raw=False, save_uint8=False):
    '''
    将图像转换为.raw格式。

    参数：
        src (str)：源图像文件夹的路径。
        dst (str)：目标图像文件夹的路径。
        req_bgr_raw (bool): 是否需要将图像数据转换为BGR格式。默认为False。
        save_uint8 (bool): 是否将图像数据保存为8位整数类型(uint8)。默认为False。

    返回：
        None

    '''
    print("Converting images to .raw")
    for root, dirs, files in os.walk(src):
        for jpgs in files:
            src_image=os.path.join(root, jpgs)
            if ('.jpg' in src_image):
                print(src_image)
                __create_raw_from_jpg(src_image, dst, req_bgr_raw, save_uint8)

def main():
    parser = argparse.ArgumentParser(description="Convert images to .raw", formatter_class=argparse.RawTextHelpFormatter)
    parser.add_argument('-s', '--src', type=str, required=True, help='Source folder for images')
    parser.add_argument('-d', '--dst', type=str, required=True, help='Destination folder for converted images')
    parser.add_argument('--req_bgr_raw', action='store_true', help='Convert image data to BGR format')
    parser.add_argument('--save_uint8', action='store_true', help='Save image data as uint8 format')

    args = parser.parse_args()

    src = os.path.abspath(args.src)
    dst = os.path.abspath(args.dst)
    req_bgr_raw = args.req_bgr_raw
    save_uint8  = args.save_uint8

    convert_images_to_raw(src, dst, req_bgr_raw, save_uint8)

if __name__ == '__main__':
    exit(main())


from django.db import transaction
from django.db.models import Q
from django.http import Http404
from django.shortcuts import get_object_or_404
from rest_framework import mixins, permissions, status
from rest_framework.decorators import list_route
from rest_framework.exceptions import ValidationError
from rest_framework.response import Response

from ...users.serializers.userSerializer import UserDataSerializer
from ...utilities import ListModelViewMixin, GenericViewSet
from ..models import (
    CRFTemplate,
    CRFTemplateQuestion,
    PatientChartReview,
    PatientChartReviewQuestion,
    ProjectCohortPatient
)

from ..serializers import (
    CohortDataSerializer,
    CRFTemplateDetailSerializer,
    CRFTemplateQuestionDataSerializer,
    PatientChartReviewCreateSerializer,
    PatientChartReviewListSerializer,
    PatientChartReviewQuestionRetrieveSerializer,
    PatientChartReviewRetrieveSerializer,
    PatientChartReviewSerializer,
    PatientChartReviewStatusSerializer,
    PatientDataSerializer,
    ProjectDataSeralizer
)

from ..serializers.patient_chart_review import ProjectChartReviewDataSerializer

class PatientChartReviewView(GenericViewSet,
                             mixins.RetrieveModelMixin,
                             ListModelViewMixin):
    queryset = PatientChartReview.objects.filter(is_active=True)
    permission_classes = (permissions.IsAuthenticated,)
    serializer_class = PatientChartReviewSerializer
    model = PatientChartReview
    curator_allowed_actions = [
        'get_completed_chart_reviews',
        'answers',
        'update_chart_review',
        'get_chart_review_status',
        'create']

    def _get_object(self, pk):
        try:
            return PatientChartReview.objects.get(pk=pk)
        except PatientChartReview.DoesNotExist:
            raise Http404

    def get_serializer_class(self):
        if self.action in ['create', 'update']:
            return PatientChartReviewCreateSerializer
        if self.action == 'get_completed_chart_reviews':
            return PatientChartReviewListSerializer
        if self.action == 'get_completed_project_chart_reviews':
            return ProjectChartReviewDataSerializer
        return PatientChartReviewSerializer

    def filter_query_set(self, search_param):
        query = Q(status__icontains=search_param)
        return self.get_queryset().filter(query)

    @list_route(methods=['get'])
    def answers(self, request):
        reqParams = request.GET
        project_id = reqParams.get('project_id', None)
        patient_id = reqParams.get('patient_id', None)
        crfTemplate_id = reqParams.get('crfTemplate_id', None)
        curator_id = reqParams.get('curator_id', request.user.id)

        try:
            patient = PatientChartReview.objects.get(
                project_id=project_id, patient_id=patient_id, curator_id=curator_id, crf_template_id=crfTemplate_id)
            patient = PatientChartReviewRetrieveSerializer(patient)
            return Response(patient.data, status=status.HTTP_200_OK)
        except PatientChartReview.DoesNotExist:
            return Response({}, status=status.HTTP_200_OK)

    def get_template_question_response(self, chart_review, template_question):
        try:
            question_response = PatientChartReviewQuestion.objects.get(
                patient_chart_review_id=chart_review.id,
                crf_template_question_id=template_question.id)
            question_response = PatientChartReviewQuestionRetrieveSerializer(
                question_response).data
            return question_response['responses']
        except PatientChartReviewQuestion.DoesNotExist:
            return None

    def get_chart_review_crf_template_data(self, chart_review):
        crf_template = chart_review.crf_template
        template_questions = crf_template.questions.all()
        questions = []
        for template_question in template_questions:
            question = CRFTemplateQuestionDataSerializer(
                template_question).data
            question['curation_response'] = self.get_template_question_response(
                chart_review, template_question)
            questions.append(question)
        crf_template_data = CRFTemplateDetailSerializer(crf_template).data
        crf_template_data['questions'] = questions
        return crf_template_data

    def get_chart_review_data(self, chart_review):
        response_data = {}
        chart_review_data = chart_review
        response_data['project'] = ProjectDataSeralizer(
            chart_review_data.project).data
        response_data['cohort'] = CohortDataSerializer(
            chart_review_data.cohort).data
        response_data['patient'] = PatientDataSerializer(
            chart_review_data.patient).data
        response_data['curator'] = UserDataSerializer(
            chart_review_data.curator).data
        response_data['status'] = chart_review_data.status
        crf_templates = []
        crf_templates.append(self.get_chart_review_crf_template_data(chart_review))
        response_data['crf_templates'] = crf_templates
        return response_data

    @list_route(methods=['get'])
    def get_chart_review_response(self, request):
        reqParams = request.GET
        project_id = reqParams.get('project_id', None)
        cohort_id = reqParams.get('cohort_id', None)
        patient_id = reqParams.get('patient_id', None)
        crfTemplate_id = reqParams.get('crf_template_id', None)
        # curator_id = reqParams.get('curator_id', None)

        try:
            queryset = PatientChartReview.objects.filter(project_id=project_id,
                                                         cohort_id=cohort_id)
            if crfTemplate_id is not None:
                queryset = queryset.filter(crf_template_id=crfTemplate_id)
            if patient_id is not None:
                queryset = queryset.filter(patient__patient_id=patient_id)
            if request.user.groups.first().name == 'curator':
                queryset = queryset.filter(curator_id=request.user.id)
            completed_chart_reviews = []
            for chart_review in queryset:
                completed_chart_reviews.append(self.get_chart_review_data(chart_review))

            return Response(completed_chart_reviews, status=status.HTTP_200_OK)
        except PatientChartReview.DoesNotExist:
            return Response({}, status=status.HTTP_200_OK)

    @list_route(methods=['get'])
    def get_chart_review_status(self, request):
        reqParams = request.GET
        project_id = reqParams.get('project_id', None)
        patient_id = reqParams.get('patient_id', None)
        curator_id = reqParams.get('curator_id', request.user.id)

        try:
            chart_review_data = PatientChartReview.objects.filter(
                project_id=project_id, patient_id=patient_id, curator_id=curator_id)
            chart_review = PatientChartReviewStatusSerializer(
                chart_review_data, many=True)
            return Response(chart_review.data, status=status.HTTP_200_OK)
        except PatientChartReview.DoesNotExist as er:
            return Response({'errors': er}, status=status.HTTP_404_NOT_FOUND)

    @transaction.atomic
    def create(self, request):
        data = request.data
        serializer = self.get_serializer(data=data)
        if not serializer.is_valid():
            return Response({'errors': serializer.errors}, status=status.HTTP_400_BAD_REQUEST)

        # validate the question belongs to template or not.
        crf_template_id = data.get('crf_template_id')
        question_responses = data.get('question_responses')

        try:
            crf_template = CRFTemplate.objects.get(id=crf_template_id)
            for question_response in question_responses:
                crf_template_question_id = question_response.get(
                    'crf_template_question_id')
                crf_template.questions.get(id=crf_template_question_id)
        except (CRFTemplate.DoesNotExist, CRFTemplateQuestion.DoesNotExist):
            error_message = "crf_template_question_id does not exists in CRF Template crf_template_id: {crf_template_id}"
            raise ValidationError(
                {'errors': error_message.format(crf_template_id=crf_template_id)})

        patient = serializer.validated_data['patient_id']
        project = serializer.validated_data['project_id']
        cohort = serializer.validated_data['cohort_id']
        curator = serializer.validated_data['curator']

        project_cohort_patient = get_object_or_404(ProjectCohortPatient,
                                                   patient_id=patient.id,
                                                   project_id=project.id,
                                                   cohort_id=cohort.id,
                                                   is_active=True)

        if project_cohort_patient.curation_status != ProjectCohortPatient.CurationStatus.completed:
            project_cohort_patient.curation_status = ProjectCohortPatient.CurationStatus.inprogress
            project_cohort_patient.curator_id = curator
            project_cohort_patient.save()

        created_obj = serializer.save()
        chart_review = PatientChartReviewRetrieveSerializer(created_obj)
        return Response(chart_review.data, status=status.HTTP_201_CREATED)

    @transaction.atomic
    def update(self, request, pk=None):
        """
        Update the status of the chart review
        """
        data = request.data
        instance = self._get_object(pk)
        serializer = self.get_serializer(instance, data=data)
        if not serializer.is_valid():
            return Response({'errors': serializer.errors}, status=status.HTTP_400_BAD_REQUEST)

        updated_obj = serializer.save()
        chart_review = PatientChartReviewRetrieveSerializer(updated_obj)
        return Response(chart_review.data, status=status.HTTP_200_OK)

    @transaction.atomic
    @list_route(methods=['POST'])
    def update_chart_review(self, request):
        """
        Update the status of the chart review
        """
        data = request.data
        project_id = data.get('project_id', None)
        patient_id = data.get('patient_id', None)
        crf_template_id = data.get('crf_template_id', None)
        cohort_id = data.get('cohort_id', None)
        instance = None
        try:
            instance = PatientChartReview.objects.get(
                project_id=project_id, patient_id=patient_id, crf_template_id=crf_template_id, cohort_id=cohort_id)
        except PatientChartReview.DoesNotExist:
            raise Http404
        serializer = self.get_serializer(instance, data=data)
        if not serializer.is_valid():
            return Response({'errors': serializer.errors}, status=status.HTTP_400_BAD_REQUEST)

        updated_obj = serializer.save()
        chart_review = PatientChartReviewRetrieveSerializer(updated_obj)
        return Response(chart_review.data, status=status.HTTP_200_OK)

    def search_chart_review_data(self, project, cohort, patient, status, user):
        """
        Searches completed chart review data
        """
        queryset = ProjectCohortPatient.objects.filter(
            curation_status=status
        ).order_by('patient', '-updated_on')

        if GenericViewSet._is_curator(user):
            # get the curator's project details from queryset if the request user is curator.
            queryset = queryset.filter(curator_id=user.id)

        if project is not None:
            queryset = queryset.filter(Q(project__name__icontains=project))
        if cohort is not None:
            queryset = queryset.filter(Q(cohort__name__icontains=cohort))
        if patient is not None:
            queryset = queryset.filter(
                Q(patient__patient_id__icontains=patient))

        # queryset = queryset.distinct('patient')
        return queryset

    @list_route(methods=['get'])
    def get_completed_chart_reviews(self, request):
        """
        Gets all completed patient chart review data based on serach criteria.
        """
        req_params = request.GET
        project = req_params.get('project', None)
        cohort = req_params.get('cohort', None)
        patient_id = req_params.get('patient_id', None)
        sort_name = req_params.get('sortName', default=None)
        sort_order = req_params.get('sortOrder', default='asc')
        status = req_params.get(
            'status', default=ProjectCohortPatient.CurationStatus.completed)

        queryset = self.search_chart_review_data(
            project, cohort, patient_id, status, request.user)

        if sort_name is not None and sort_order is not None:
            queryset = self.sort_chart_reivews(queryset, sort_name, sort_order)

        page = self.paginate_queryset(queryset)
        data = []

        if page is not None:
            serializer = self.get_serializer(page, many=True)
            data = serializer.data

        return self.get_paginated_response(data)

    def sort_chart_reivews(self, queryset, sort_name, sort_order):
        """
        Sorts the chart review data by cohort name, project name, curator email
        """
        if sort_name == 'cohort_name':
            sort_name = 'cohort__name'
        if sort_name == 'project_name':
            sort_name = 'project__name'
        if sort_name == 'curator_email':
            sort_name = 'curator__email'

        if sort_order == 'desc':
            sort_name = '-{}'.format(sort_name)
        return queryset.order_by(sort_name)

    def search_project_chart_review_data(self, project, cohort, crf_template, user, search_param):
        """
        Searches completed project chart review data
        """
        queryset = ProjectCohortPatient.objects.filter(
            curation_status=ProjectCohortPatient.CurationStatus.completed
        ).order_by('patient_id', '-updated_on')

        if GenericViewSet._is_curator(user):
            # get the curator's project details from queryset if the request user is curator.
            queryset = queryset.filter(curator_id=user.id)

        if project is not None:
            queryset = queryset.filter(Q(project__name__icontains=project))
        if cohort is not None:
            queryset = queryset.filter(Q(cohort__name__icontains=cohort))

        cohort_ids = queryset.values_list('cohort_id', flat=True)
        project_ids = queryset.values_list('project_id', flat=True)

        compelted_crf_templates = PatientChartReview.objects.filter(
            status=PatientChartReview.StatusType.completed,
            is_active=True,
            cohort_id__in=cohort_ids,
            project_id__in=project_ids).distinct('crf_template')

        if crf_template is not None:
            compelted_crf_templates = compelted_crf_templates.filter(
                Q(crf_template__name__icontains=crf_template))

        if search_param is not None:
            compelted_crf_templates = compelted_crf_templates.filter(Q(cohort__name__icontains=search_param) | Q(
                project__name__icontains=search_param) | Q(
                    crf_template__name__icontains=search_param
                ))

        # queryset = queryset.distinct('patient')
        return compelted_crf_templates

    def sort_project_chart_reivews(self, queryset, sort_name, sort_order):
        """
        Sorts the chart review data by project name, crf template name
        """
        if sort_name == 'project':
            sort_name = 'project__name'
        if sort_name == 'cohort':
            sort_name = 'cohort__name'
        if sort_name == 'crf_template':
            sort_name = 'crf_template__name'

        if sort_order == 'desc':
            sort_name = '-{}'.format(sort_name)
        return queryset.order_by(sort_name)

    @list_route(methods=['get'])
    def get_completed_project_chart_reviews(self, request):
        """
        Gets all completed patient chart review per project CRF template based on serach criteria.
        """
        req_params = request.GET
        project = req_params.get('project', None)
        cohort = req_params.get('cohort', None)
        crf_template = req_params.get('crf_template', None)
        search_param = req_params.get('searchParam', default=None)
        sort_name = req_params.get('sortName', default=None)
        sort_order = req_params.get('sortOrder', default='asc')

        queryset = self.search_project_chart_review_data(project, cohort, crf_template, request.user, search_param)

        if sort_name is not None and sort_order is not None:
            queryset = self.sort_chart_reivews(queryset, sort_name, sort_order)

        page = self.paginate_queryset(queryset)
        data = []

        if page is not None:
            serializer = self.get_serializer(page, many=True)
            data = serializer.data

        return self.get_paginated_response(data)


from django import forms
from django.utils.translation import gettext as _
from django.template.loader import render_to_string
from django.contrib.admin.widgets import FilteredSelectMultiple
from bootstrap_modal_forms.forms import BSModalModelForm, BSModalForm
from constance import config

from utils.scraping import xml_parser, xml_sivin_parser
from utils.django.validators import validate_vin, vin_psa_isvalid
from utils.file.export import xml_corvet_file
from utils.conf import string_to_list
from psa.models import Corvet, Multimedia, Ecu
from .models import Xelon, Action, Sivin, ProductCode, ProductCategory
from .tasks import send_email_task
from utils.django.forms.fields import ListTextWidget
from utils.django.models import defaults_dict
from utils.file import LogFile
from utils.conf import CSD_ROOT

class IhmEmailModalForm(BSModalForm):
    to = forms.CharField(label='À', required=True, widget=forms.TextInput())
    cc = forms.CharField(label='Cc', widget=forms.TextInput())
    subject = forms.CharField(label='Objet', required=True, widget=forms.TextInput())
    message = forms.CharField(widget=forms.Textarea(attrs={'rows': 10}), required=True)

    def __init__(self, *args, **kwargs):
        super(IhmEmailModalForm, self).__init__(*args, **kwargs)
        cc_email_list = config.CSD_CC_EMAIL_LIST
        if self.request.user.email not in cc_email_list:
            cc_email_list = f"{self.request.user.email}; {cc_email_list}"
        self.fields['cc'].initial = cc_email_list

    def send_email(self):
        send_email_task.delay(
            subject=self.cleaned_data['subject'], body=self.cleaned_data['message'], from_email=self.request.user.email,
            to=string_to_list(self.cleaned_data['to']), cc=string_to_list(self.cleaned_data['cc'])
        )

    @staticmethod
    def vin_message(model, request):
        domain = config.WEBSITE_DOMAIN
        queryset = model.actions.filter(content__contains="OLD_VIN")
        rasp_log = LogFile(CSD_ROOT).vin_err_filter(model.modele_produit, model.numero_de_dossier)
        if queryset:
            data = queryset.first().content.split('\n')
            vins = {"old_vin": data[0][-17:], "new_vin": data[1][-17:]}
        else:
            vins = None
        message = render_to_string('squalaetp/email_format/vin_error_email.html', locals())
        return message

    @staticmethod
    def prod_message(model, request):
        domain = config.WEBSITE_DOMAIN
        queryset = model.actions.filter(content__contains="OLD_PROD")
        if queryset:
            data = queryset.first().content.split('\n')
            prods = {"old_prod": data[0][9:], "new_prod": data[1][9:]}
        else:
            prods = None
        message = render_to_string('squalaetp/email_format/prod_error_email.html', locals())
        return message

    @staticmethod
    def adm_message(model, request):
        domain = config.WEBSITE_DOMAIN
        if request.GET.get("select") == "prod":
            select = "modèle produit"
        else:
            select = "V.I.N."
        message = render_to_string('squalaetp/email_format/adm_email.html', locals())
        return message

class VinCorvetModalForm(BSModalModelForm):
    xml_data = forms.CharField(
        widget=forms.Textarea(
            attrs={
                'class': 'form-control',
                'placeholder': _("Data in XML format available on the RepairNAV site during CORVET extraction..."),
                'rows': 10,
            }
        ),
        required=False
    )

    class Meta:
        model = Xelon
        fields = ['vin']
        widgets = {
            'vin': forms.TextInput(
                attrs={'class': 'form-control', 'placeholder': _("Enter the VIN number"), 'autofocus': ''}
            ),
        }

    def __init__(self, *args, **kwargs):
        super(VinCorvetModalForm, self).__init__(*args, **kwargs)
        self.oldVin = self.instance.vin

    def clean_vin(self):
        data = self.cleaned_data['vin']
        message = validate_vin(data)
        if message:
            raise forms.ValidationError(_(message), code='invalid', params={'value': data})
        return data

    def clean_xml_data(self):
        xml_data = self.cleaned_data['xml_data']
        if xml_data:
            data = xml_parser(xml_data)
            no_fields = ['vin', 'btel', 'radio', 'bsi', 'emf', 'cmm', 'bsm']
            all_data = {key: '' for key in [f.name for f in Corvet._meta.local_fields if f.name not in no_fields]}
            all_data.update({'donnee_date_debut_garantie': None, 'donnee_date_entree_montage': None})
            vin = self.cleaned_data.get("vin")
            if isinstance(data, dict):
                all_data.update(data)
                if data.get('vin') == vin and data.get('donnee_date_entree_montage'):
                    if self.request.is_ajax():
                        xml_corvet_file(self.instance, xml_data, vin)
                if data.get('vin') != vin:
                    self.add_error('xml_data', _('XML data does not match VIN'))
            else:
                self.add_error('xml_data', _('Invalid XML data'))
                all_data = data
            return all_data
        return xml_data

    def clean(self):
        cleaned_data = super(VinCorvetModalForm, self).clean()
        vin = cleaned_data.get('vin')
        data = cleaned_data.get('xml_data')
        if vin and isinstance(data, dict) and not data.get('donnee_date_entree_montage'):
            raise forms.ValidationError(_('VIN error !'))
        elif vin != self.oldVin and vin_psa_isvalid(vin) and not isinstance(data, dict):
            if not Corvet.objects.filter(vin=vin):
                raise forms.ValidationError(_('New PSA VIN, please use the Import CORVET button !'))

    def save(self, commit=True):
        instance = super().save(commit=False)
        if commit and not self.request.is_ajax():
            data = self.cleaned_data['xml_data']
            vin = self.cleaned_data['vin']
            if data and vin:
                defaults = defaults_dict(Corvet, data, 'vin')
                Corvet.objects.update_or_create(vin=vin, defaults=defaults)
            if vin != self.oldVin:
                content = "OLD_VIN: {}\nNEW_VIN: {}".format(self.oldVin, vin)
                Action.objects.create(content=content, content_object=self.instance)
            instance.save()
        return instance

class ProductModalForm(BSModalModelForm):
    class Meta:
        model = Xelon
        fields = ['modele_produit', 'modele_vehicule']
        widgets = {'modele_vehicule': forms.TextInput(attrs={'readonly': True})}

    def __init__(self, *args, **kwargs):
        products = ProductCategory.objects.exclude(product_model="").order_by('product_model')
        self.data_list = list(products.values_list('product_model', flat=True).distinct())
        super(ProductModalForm, self).__init__(*args, **kwargs)
        self.fields['modele_produit'].widget = ListTextWidget(data_list=self.data_list, name='value-list')
        self.fields['modele_produit'].required = True
        self.oldProduct = self.instance.modele_produit
        self.oldVehicle = self.instance.modele_vehicule

    def clean_modele_produit(self):
        data = self.cleaned_data['modele_produit']
        if data not in self.data_list:
            self.add_error('modele_produit', _('Xelon product model does not exist !'))
        return data

    def save(self, commit=True):
        instance = super().save(commit=False)
        if commit and not self.request.is_ajax():
            product = self.cleaned_data['modele_produit']
            vehicle = self.cleaned_data['modele_vehicule']
            if product and vehicle and not self.request.is_ajax():
                if product != self.oldProduct:
                    content = "OLD_PROD: {}\nNEW_PROD: {}".format(self.oldProduct, product)
                    Action.objects.create(content=content, content_object=self.instance)
                if vehicle != self.oldVehicle:
                    content = "OLD_VEH: {}\nNEW_VEH: {}".format(self.oldProduct, vehicle)
                    Action.objects.create(content=content, content_object=self.instance)
            instance.save()
        return instance

class SivinModalForm(BSModalModelForm):
    immat_siv = forms.CharField(
        widget=forms.TextInput(
            attrs={'class': 'form-control', 'placeholder': _("Enter the IMMAT number"), 'autofocus': ''}
        )
    )
    xml_data = forms.CharField(
        widget=forms.Textarea(
            attrs={
                'class': 'form-control',
                'placeholder': _("Data in XML format available on the RepairNAV site during SIVIN extraction..."),
                'rows': 10,
            }
        ),
        required=True
    )

    class Meta:
        model = Sivin
        exclude = ["corvet"]

    def __init__(self, *args, **kwargs):
        super().__init__(*args, **kwargs)
        self.fields['codif_vin'].required = False

    def clean_xml_data(self):
        xml_data = self.cleaned_data['xml_data']
        immat_siv = self.cleaned_data.get('immat_siv', '').upper()
        data = xml_sivin_parser(xml_data)
        if isinstance(data, dict):
            if data.get('immat_siv') == immat_siv:
                for field, value in data.items():
                    self.cleaned_data[field] = value
                return xml_data
            self.add_error('xml_data', _('XML data does not match IMMAT'))
        else:
            self.add_error('xml_data', _('Invalid XML data'))
        return xml_data

    def save(self, commit=True):
        del self.fields['xml_data']
        instance = super().save(commit=False)
        if commit and not self.request.is_ajax():
            instance.save()
        return instance

class ProductCodeAdminForm(forms.ModelForm):
    medias = forms.ModelMultipleChoiceField(
        queryset=Multimedia.objects.all(), widget=FilteredSelectMultiple("Multimedia", is_stacked=False),
        required=False)
    ecus = forms.ModelMultipleChoiceField(
        queryset=Ecu.objects.all(), widget=FilteredSelectMultiple("Ecu", is_stacked=False), required=False)

    class Meta:
        model = ProductCode
        fields = ['name', 'medias', 'ecus']

class XelonCloseModalForm(BSModalModelForm):

    class Meta:
        model = Xelon
        fields = ['type_de_cloture']

    def save(self, commit=True):
        instance = super().save(commit=False)
        if commit and not self.request.is_ajax():
            instance.type_de_cloture = "N/A"
            instance.save()
            Action.objects.create(content="Dossier en retard => FAIT", content_object=instance)
        return instance


import json
from bs4 import BeautifulSoup
import requests

focada = False

# Variáveis de tratamento de URL
urlbase = "https://www.vagas.com.br/vagas-de-"
urlVaga = "https://www.vagas.com.br"
palavras_chaves = [
    "médico", "marceneiro", "geral", "ajudante", "dentista",
    "clínico", "vendedor", "transporte", "aprendiz", "soldador",
    "servente", "obras", "administrativo", "nutricionista", "eletricista",
    "confeiteiro", "operador", "cobrador", "almoxarifado", "vigilante", "designer",
    "químico", "administração", "mecânico", "logística", "segurança", "gerente",
    "manicure", "pedicure", "estoquista", "professor", "limpeza", "fiscal",
    "analista", "entregador", "motorista", "serralheiro", "suporte", "telemarketing",
    "frentista"
]
# Variável de armazenamento de um 'dicionário' de vagas
dict_vagas = []

# Arquivo JSON que armazenará cada vaga
arquivo = "./web-crawler/vagas-geral/vagas-geral.json"

def remove(dictionary):
	for key, value in dictionary.items():
		if isinstance(value, dict):
			remove(value)
		else:
			if isinstance(dictionary[key], str):
				dictionary[key] = value.strip()

for palavra in palavras_chaves:
    urlContent = requests.get(urlbase+palavra).content
    interpretedHtml = BeautifulSoup(urlContent, 'html.parser')

    for htmlVagas in interpretedHtml.select('li.vaga')[:30]:
        # Raspagem do título das vagas
        tituloVaga = htmlVagas.find('a', class_='link-detalhes-vaga').text
        tituloVaga = tituloVaga.split()
        tituloVaga = ' '.join(tituloVaga)

        # Raspagem da empresa das vagas
        empresaVaga = htmlVagas.find('span', class_='emprVaga').text
        empresaVaga = empresaVaga.split()
        empresaVaga = ' '.join(empresaVaga)

        # Raspagem do nível técnico das vagas
        nivelVaga = htmlVagas.find('span', class_='nivelVaga').text
        nivelVaga = nivelVaga.split()
        nivelVaga = ' '.join(nivelVaga)

        # Raspagem da localização das vagas
        if htmlVagas.find('span', class_='vaga-local') is not None:
            localizacaoVaga = htmlVagas.find('span', class_='vaga-local').text
            localizacaoVaga = localizacaoVaga.split()
            localizacaoVaga = ' '.join(localizacaoVaga)
            localizacaoVaga = localizacaoVaga.split("/")
            cidade = localizacaoVaga[0]
            if len(localizacaoVaga) > 1:
                estado = localizacaoVaga[1]
            else:
                estado = None
        else:
            cidade = None
            estado = None

        # Request da página de cada vaga
        vagaHref = htmlVagas.find('a', class_='link-detalhes-vaga', href=True)
        vagaInfo = requests.get(urlVaga+vagaHref['href']).content
        interpretedVagaInfo = BeautifulSoup(vagaInfo, 'html.parser')

        for info in interpretedVagaInfo.find_all('article', class_="vaga job-group"):
            # Raspagem da descrição das vagas
            descricaoVaga = info.find('div', class_="job-description__text").text
            descricaoVaga = descricaoVaga.split()
            descricaoVaga = ' '.join(descricaoVaga)

            if (info.find('ul', class_="job-benefits__list") and info.find('div', class_="job-company-presentation")):
                #   Raspagem dos benefícios das vaga
                beneficiosVaga = info.find('ul', class_="job-benefits__list").text
                beneficiosVaga = beneficiosVaga.split()
                beneficiosVaga = ' '.join(beneficiosVaga)

                # Raspagem da descrição da empresa das vagas
                descricaoEmpresa = info.find('div', class_="job-company-presentation").text
                descricaoEmpresa = descricaoEmpresa.split()
                descricaoEmpresa = ' '.join(descricaoEmpresa)

                dict_vagas.append({
                    "focada": focada,
                    "categoria": palavra,
                    "titulo": tituloVaga,
                    "salario": "À combinar",
                    "empresa": empresaVaga,
                    "nivel": nivelVaga,
                    "descricao": descricaoVaga,
                    "cidade": cidade,
                    "estado": estado,
                    "beneficios": beneficiosVaga,
                    "descEmpresa": descricaoEmpresa
                })
            elif (info.find('ul', class_="job-benefits__list")):
                #   Raspagem dos benefícios das vaga
                beneficiosVaga = info.find('ul', class_="job-benefits__list").text
                beneficiosVaga = beneficiosVaga.split()
                beneficiosVaga = ' '.join(beneficiosVaga)

                dict_vagas.append({
                    "focada": focada,
                    "categoria": palavra,
                    "titulo": tituloVaga,
                    "salario": "À combinar",
                    "empresa": empresaVaga,
                    "nivel": nivelVaga,
                    "descricao": descricaoVaga,
                    "cidade": cidade,
                    "estado": estado,
                    "beneficios": beneficiosVaga,
                })
            elif (info.find('div', class_="job-company-presentation")):
                # Raspagem da descrição da empresa das vagas
                descricaoEmpresa = info.find('div', class_="job-company-presentation").text
                descricaoEmpresa = descricaoEmpresa.split()
                descricaoEmpresa = ' '.join(descricaoEmpresa)

                dict_vagas.append({
                    "focada": focada,
                    "categoria": palavra,
                    "titulo": tituloVaga,
                    "salario": "À combinar",
                    "empresa": empresaVaga,
                    "nivel": nivelVaga,
                    "descricao": descricaoVaga,
                    "cidade": cidade,
                    "estado": estado,
                    "descEmpresa": descricaoEmpresa
                })
            else:
                dict_vagas.append({
                    "focada": focada,
                    "categoria": palavra,
                    "titulo": tituloVaga,
                    "salario": "À combinar",
                    "empresa": empresaVaga,
                    "nivel": nivelVaga,
                    "descricao": descricaoVaga,
                    "cidade": cidade,
                    "estado": estado,
                })

with open(arquivo, mode='r', encoding='utf-8') as loadJson:
    data = json.load(loadJson)

for dataToInsert in dict_vagas:
    remove(dataToInsert)
    data.append(dataToInsert)

with open(arquivo, mode='w', encoding='utf-8') as newData:
    json.dump(data, newData, ensure_ascii=False, indent=4, separators=(',',': '))


import os
from typing import Annotated

from fastapi import Depends
from langchain.embeddings.openai import OpenAIEmbeddings
from langchain.vectorstores import SupabaseVectorStore
from logger import get_logger

from supabase import Client, create_client

logger = get_logger(__name__)

openai_api_key = os.environ.get("OPENAI_API_KEY")
anthropic_api_key = os.environ.get("ANTHROPIC_API_KEY")
supabase_url = os.environ.get("SUPABASE_URL")
supabase_key = os.environ.get("SUPABASE_SERVICE_KEY")

embeddings = OpenAIEmbeddings(openai_api_key=openai_api_key)
supabase_client: Client = create_client(supabase_url, supabase_key)
documents_vector_store = SupabaseVectorStore(
    supabase_client, embeddings, table_name="vectors")
summaries_vector_store = SupabaseVectorStore(
    supabase_client, embeddings, table_name="summaries")

def common_dependencies():
    return {
        "supabase": supabase_client,
        "embeddings": embeddings,
        "documents_vector_store": documents_vector_store,
        "summaries_vector_store": summaries_vector_store
    }

CommonsDep = Annotated[dict, Depends(common_dependencies)]


import numpy as np
from mtcnn import MTCNN
import cv2
from numpy import expand_dims
from PIL import Image
from sklearn.preprocessing import Normalizer, LabelEncoder

def detect_face(filename, detector):
	img = cv2.cvtColor(cv2.imread(filename), cv2.COLOR_BGR2RGB)
	results = detector.detect_faces(img)
	x1, y1, width, height = results[0]['box']
	x1, y1 = abs(x1), abs(y1)
	x2, y2 = x1 + width, y1 + height
	face = img[y1:y2, x1:x2]
	return face

def get_embeddings(face_pixels, model):
	#center the pixel
	new_width  = 160
	new_height = 160
	face_pixels = np.array(Image.fromarray(face_pixels).resize((new_width, new_height), Image.ANTIALIAS))
	face_pixels = face_pixels.astype('float32')
	mean, std = face_pixels.mean(), face_pixels.std()
	face_pixels = (face_pixels - mean) / std
	samples = expand_dims(face_pixels, axis=0)
	samples.shape
	# make prediction to get embedding
	yhat = model.predict(samples)
	return yhat[0]

def normalizer(trainX, testX):
	in_encoder = Normalizer(norm='l2')
	trainX = in_encoder.transform(trainX)
	testX = in_encoder.transform(testX)
	return trainX, testX

def normalizer(trainX):
	in_encoder = Normalizer(norm='l2')
	trainX = in_encoder.transform(trainX)
	return trainX

def labeller(trainy):
	out_encoder = LabelEncoder()
	out_encoder.fit(trainy)
	trainy = out_encoder.transform(trainy)
	return trainy

def labeller(trainy, testy):
	out_encoder = LabelEncoder()
	out_encoder.fit(trainy)
	trainy = out_encoder.transform(trainy)
	testy = out_encoder.transform(testy)
	return trainy, testy