"""Unit tests for the SonarQube source."""

from datetime import datetime, timedelta, timezone

from source_model import Entity

from ..source_collector_test_case import SourceCollectorTestCase

class SonarQubeTestCase(SourceCollectorTestCase):
    """Base class for the SonarQube metrics unit tests."""

    def setUp(self):
        super().setUp()
        self.sources = dict(
            source_id=dict(
                type="sonarqube", parameters=dict(url="https://sonar", component="id", types=["bug", "code_smell"])))
        self.tests_landing_url = "https://sonar/component_measures?id=id&metric=tests&branch=master"
        self.issues_landing_url = "https://sonar/project/issues?id=id&resolved=false&branch=master"
        self.issue_landing_url = "https://sonar/project/issues?id=id&issues={0}&open={0}&branch=master"
        self.hotspot_landing_url = "https://sonar/security_hotspots?id=id&hotspots={0}&branch=master"

    def entity(  # pylint: disable=too-many-arguments
            self, component: str, entity_type: str, severity: str = None, resolution: str = None,
            vulnerability_probability: str = None) -> Entity:
        """Create an entity."""
        url = self.hotspot_landing_url.format(component) if entity_type == "security_hotspot" else \
            self.issue_landing_url.format(component)
        entity = Entity(key=component, component=component, message=component, type=entity_type, url=url)
        if severity is not None:
            entity["severity"] = severity
        if resolution is not None:
            entity["resolution"] = resolution
        if vulnerability_probability is not None:
            entity["vulnerability_probability"] = vulnerability_probability
        return entity

class SonarQubeViolationsTest(SonarQubeTestCase):
    """Unit tests for the SonarQube violations metric."""

    def setUp(self):
        super().setUp()
        self.metric = dict(type="violations", addition="sum", sources=self.sources)

    async def test_violations(self):
        """Test that the number of violations is returned."""
        json = dict(
            total="2",
            issues=[
                dict(key="a", message="a", component="a", severity="INFO", type="BUG"),
                dict(key="b", message="b", component="b", severity="MAJOR", type="CODE_SMELL")])
        response = await self.collect(self.metric, get_request_json_return_value=json)
        expected_entities = [self.entity("a", "bug", "info"), self.entity("b", "code_smell", "major")]
        self.assert_measurement(response, value="2", entities=expected_entities, landing_url=self.issues_landing_url)

class SonarQubeTestsTest(SonarQubeTestCase):
    """Unit tests for the SonarQube tests metric."""

    def setUp(self):
        super().setUp()
        self.metric = dict(type="tests", addition="sum", sources=self.sources)

    async def test_nr_of_tests(self):
        """Test that the number of tests is returned."""
        json = dict(component=dict(measures=[dict(metric="tests", value="123")]))
        response = await self.collect(self.metric, get_request_json_return_value=json)
        self.assert_measurement(response, value="123", total="123", landing_url=self.tests_landing_url)

    async def test_nr_of_skipped_tests(self):
        """Test that the number of skipped tests is returned."""
        json = dict(
            component=dict(measures=[dict(metric="tests", value="123"), dict(metric="skipped_tests", value="4")]))
        self.sources["source_id"]["parameters"]["test_result"] = ["skipped"]
        response = await self.collect(self.metric, get_request_json_return_value=json)
        self.assert_measurement(response, value="4", total="123", landing_url=self.tests_landing_url)

    async def test_nr_of_tests_without_tests(self):
        """Test that the collector throws an exception if there are no tests."""
        json = dict(component=dict(measures=[]))
        response = await self.collect(self.metric, get_request_json_return_value=json)
        self.assert_measurement(
            response, value=None, total=None, parse_error="KeyError", landing_url=self.tests_landing_url)

    async def test_nr_of_tests_with_faulty_component(self):
        """Test that the measurement fails if the component does not exist."""
        response = await self.collect(
            self.metric, get_request_json_return_value=dict(errors=[dict(msg="No such component")]))
        self.assert_measurement(
            response, value=None, total=None, connection_error="No such component", landing_url=self.tests_landing_url)

class SonarQubeMetricsTest(SonarQubeTestCase):
    """Unit tests for the other SonarQube metrics."""

    def setUp(self):
        super().setUp()
        self.metric_landing_url = "https://sonar/component_measures?id=id&metric={0}&branch=master"
        self.all_code_smells = "effort to fix all code smells"
        self.all_bug_issues = "effort to fix all bug issues"

    async def test_commented_out_code(self):
        """Test that the number of lines with commented out code is returned."""
        json = dict(total="2")
        metric = dict(type="commented_out_code", addition="sum", sources=self.sources)
        response = await self.collect(metric, get_request_json_return_value=json)
        self.assert_measurement(
            response, value="2", total="100",
            landing_url=f"{self.issues_landing_url}&rules=abap:S125,apex:S125,c:CommentedCode,cpp:CommentedCode,"
                        "flex:CommentedCode,csharpsquid:S125,javascript:CommentedCode,javascript:S125,kotlin:S125,"
                        "objc:CommentedCode,php:S125,plsql:S125,python:S125,scala:S125,squid:CommentedOutCodeLine,"
                        "java:S125,swift:S125,typescript:S125,Web:AvoidCommentedOutCodeCheck,xml:S125")

    async def test_complex_units(self):
        """Test that the number of lines with commented out code is returned."""
        complex_units_json = dict(total="2")
        functions_json = dict(component=dict(measures=[dict(metric="functions", value="4")]))
        metric = dict(type="complex_units", addition="sum", sources=self.sources)
        response = await self.collect(
            metric,
            get_request_json_side_effect=[
                {}, complex_units_json, functions_json, complex_units_json, functions_json, complex_units_json])
        self.assert_measurement(
            response, value="2", total="4",
            landing_url=f"{self.issues_landing_url}&rules=csharpsquid:S1541,csharpsquid:S3776,flex:FunctionComplexity,"
                        "javascript:FunctionComplexity,javascript:S1541,javascript:S3776,go:S3776,kotlin:S3776,"
                        "php:S1541,php:S3776,python:FunctionComplexity,python:S3776,ruby:S3776,scala:S3776,"
                        "squid:MethodCyclomaticComplexity,java:S1541,squid:S3776,typescript:S1541,typescript:S3776,"
                        "vbnet:S1541,vbnet:S3776")

    async def test_tests(self):
        """Test that the number of tests is returned."""
        json = dict(component=dict(measures=[dict(metric="tests", value="88")]))
        metric = dict(type="tests", addition="sum", sources=self.sources)
        response = await self.collect(metric, get_request_json_return_value=json)
        self.assert_measurement(response, value="88", landing_url=self.tests_landing_url)

    async def test_uncovered_lines(self):
        """Test that the number of uncovered lines and the number of lines to cover are returned."""
        json = dict(
            component=dict(
                measures=[dict(metric="uncovered_lines", value="100"), dict(metric="lines_to_cover", value="1000")]))
        metric = dict(type="uncovered_lines", addition="sum", sources=self.sources)
        response = await self.collect(metric, get_request_json_return_value=json)
        self.assert_measurement(
            response, value="100", total="1000", landing_url=self.metric_landing_url.format("uncovered_lines"))

    async def test_uncovered_branches(self):
        """Test that the number of uncovered branches and the number of branches to cover are returned."""
        json = dict(
            component=dict(
                measures=[
                    dict(metric="uncovered_conditions", value="10"), dict(metric="conditions_to_cover", value="200")]))
        metric = dict(type="uncovered_branches", addition="sum", sources=self.sources)
        response = await self.collect(metric, get_request_json_return_value=json)
        self.assert_measurement(
            response, value="10", total="200", landing_url=self.metric_landing_url.format("uncovered_conditions"))

    async def test_duplicated_lines(self):
        """Test that the number of duplicated lines and the total number of lines are returned."""
        json = dict(
            component=dict(
                measures=[
                    dict(metric="duplicated_lines", value="10"), dict(metric="lines", value="100")]))
        metric = dict(type="duplicated_lines", addition="sum", sources=self.sources)
        response = await self.collect(metric, get_request_json_return_value=json)
        self.assert_measurement(
            response, value="10", total="100", landing_url=self.metric_landing_url.format("duplicated_lines"))

    async def test_many_parameters(self):
        """Test that the number of functions with too many parameters is returned."""
        self.sources["source_id"]["parameters"]["rules"] = ["rule1"]
        many_parameters_json = dict(total="2", issues=[])
        functions_json = dict(component=dict(measures=[dict(metric="functions", value="4")]))
        metric = dict(type="many_parameters", addition="sum", sources=self.sources)
        response = await self.collect(
            metric,
            get_request_json_side_effect=[
                {}, many_parameters_json, functions_json, many_parameters_json, functions_json, many_parameters_json])
        self.assert_measurement(
            response, value="2", total="4",
            landing_url=f"{self.issues_landing_url}&rules=c:S107,csharpsquid:S107,csharpsquid:S2436,cpp:S107,flex:S107,"
                        "javascript:ExcessiveParameterList,javascript:S107,objc:S107,php:S107,"
                        "plsql:PlSql.FunctionAndProcedureExcessiveParameters,python:S107,squid:S00107,java:S107,"
                        "tsql:S107,typescript:S107")

    async def test_long_units(self):
        """Test that the number of long units is returned."""
        self.sources["source_id"]["parameters"]["rules"] = ["rule1"]
        long_units_json = dict(total="2", issues=[])
        functions_json = dict(component=dict(measures=[dict(metric="functions", value="4")]))
        metric = dict(type="long_units", addition="sum", sources=self.sources)
        response = await self.collect(
            metric, get_request_json_side_effect=[
                {}, long_units_json, functions_json, long_units_json, functions_json, long_units_json])
        self.assert_measurement(
            response, value="2", total="4",
            landing_url=f"{self.issues_landing_url}&rules=abap:S104,c:FileLoc,cpp:FileLoc,csharpsquid:S104,"
                        "csharpsquid:S138,flex:S138,go:S104,go:S138,javascript:S104,javascript:S138,kotlin:S104,"
                        "kotlin:S138,objc:FileLoc,php:S104,php:S138,php:S2042,Pylint:R0915,python:S104,ruby:S104,"
                        "ruby:S138,scala:S104,scala:S138,squid:S00104,squid:S1188,squid:S138,java:S138,squid:S2972,"
                        "swift:S104,typescript:S104,typescript:S138,vbnet:S104,vbnet:S138,Web:FileLengthCheck,"
                        "Web:LongJavaScriptCheck")

    async def test_source_up_to_dateness(self):
        """Test that the number of days since the last analysis is returned."""
        json = dict(analyses=[dict(date="2019-03-29T14:20:15+0100")])
        metric = dict(type="source_up_to_dateness", addition="max", sources=self.sources)
        response = await self.collect(metric, get_request_json_return_value=json)
        timezone_info = timezone(timedelta(hours=1))
        expected_age = (datetime.now(timezone_info) - datetime(2019, 3, 29, 14, 20, 15, tzinfo=timezone_info)).days
        self.assert_measurement(
            response, value=str(expected_age), landing_url="https://sonar/project/activity?id=id&branch=master")

    async def test_suppressed_violations(self):
        """Test that the number of suppressed violations includes both suppressed issues as well as suppressed rules."""
        violations_json = dict(
            total="1", issues=[dict(key="a", message="a", component="a", severity="INFO", type="BUG")])
        wont_fix_json = dict(
            total="1",
            issues=[
                dict(key="b", message="b", component="b", severity="MAJOR", type="CODE_SMELL", resolution="WONTFIX")])
        total_violations_json = dict(total="4")
        metric = dict(type="suppressed_violations", addition="sum", sources=self.sources)
        response = await self.collect(
            metric, get_request_json_side_effect=[{}, violations_json, wont_fix_json, total_violations_json])
        expected_entities = [self.entity("a", "bug", "info", ""), self.entity("b", "code_smell", "major", "won't fix")]
        self.assert_measurement(
            response, value="2", total="4", entities=expected_entities,
            landing_url=f"{self.issues_landing_url}&rules=csharpsquid:S1309,php:NoSonar,Pylint:I0011,Pylint:I0020,"
                        "squid:NoSonar,java:NoSonar,squid:S1309,java:S1309,squid:S1310,"
                        "java:S1310,squid:S1315,java:S1315")

    async def test_loc_returns_ncloc_by_default(self):
        """Test that the number of lines of non-comment code is returned."""
        json = dict(
            component=dict(
                measures=[
                    dict(metric="ncloc", value="1234"),
                    dict(metric="ncloc_language_distribution", value="py=1000;js=234")]))
        metric = dict(type="loc", addition="sum", sources=self.sources)
        response = await self.collect(metric, get_request_json_return_value=json)
        self.assert_measurement(
            response, value="1234", total="100",
            entities=[
                dict(key="py", language="Python", ncloc="1000"), dict(key="js", language="JavaScript", ncloc="234")],
            landing_url=self.metric_landing_url.format("ncloc"))

    async def test_loc_all_lines(self):
        """Test that the number of lines of code is returned."""
        self.sources["source_id"]["parameters"]["lines_to_count"] = "all lines"
        json = dict(
            component=dict(
                measures=[
                    dict(metric="lines", value="1234"),
                    dict(metric="ncloc_language_distribution", value="py=999;js=10")]))
        metric = dict(type="loc", addition="sum", sources=self.sources)
        response = await self.collect(metric, get_request_json_return_value=json)
        self.assert_measurement(
            response, value="1234", total="100", entities=[], landing_url=self.metric_landing_url.format("lines"))

    async def test_remediation_effort(self):
        """Test that the remediation effort is returned, as selected by the user."""
        self.sources["source_id"]["parameters"]["effort_types"] = [self.all_code_smells, self.all_bug_issues]
        json = dict(
            component=dict(
                measures=[
                    dict(metric="reliability_remediation_effort", value="0"),
                    dict(metric="sqale_index", value="20")]))
        metric = dict(type="remediation_effort", addition="sum", sources=self.sources)
        response = await self.collect(metric, get_request_json_return_value=json)
        self.assert_measurement(
            response, value="20", total="100", entities=[
                dict(key="sqale_index", effort_type=self.all_code_smells, effort="20",
                     url=self.metric_landing_url.format("sqale_index")),
                dict(key="reliability_remediation_effort", effort_type=self.all_bug_issues, effort="0",
                     url=self.metric_landing_url.format("reliability_remediation_effort"))],
            landing_url="https://sonar/component_measures?id=id&branch=master")

    async def test_remediation_effort_one_metric(self):
        """Test that the remediation effort is returned, as selected by the user and that the landing url points to
        the metric."""
        self.sources["source_id"]["parameters"]["effort_types"] = [self.all_code_smells]
        json = dict(component=dict(measures=[dict(metric="sqale_index", value="20")]))
        metric = dict(type="remediation_effort", addition="sum", sources=self.sources)
        response = await self.collect(metric, get_request_json_return_value=json)
        self.assert_measurement(
            response, value="20", total="100", entities=[
                dict(key="sqale_index", effort_type=self.all_code_smells, effort="20",
                     url=self.metric_landing_url.format("sqale_index"))],
            landing_url=self.metric_landing_url.format("sqale_index"))

class SonarQubeSecurityWarningsTest(SonarQubeTestCase):
    """Unit tests for the SonarQube security warnings collector."""

    async def test_security_warnings(self):
        """Test that all security warnings are returned."""
        show_component_json = {}
        vulnerabilities_json = dict(
            total="2",
            issues=[
                dict(key="a", message="a", component="a", severity="INFO", type="VULNERABILITY"),
                dict(key="b", message="b", component="b", severity="MAJOR", type="VULNERABILITY")])
        hotspots_json = dict(
            paging=dict(total="2"),
            hotspots=[
                dict(key="a", message="a", component="a", vulnerabilityProbability="MEDIUM"),
                dict(key="b", message="b", component="b", vulnerabilityProbability="LOW")])
        metric = dict(type="security_warnings", addition="sum", sources=self.sources)
        response = await self.collect(
            metric, get_request_json_side_effect=[show_component_json, vulnerabilities_json, hotspots_json])
        expected_entities = [
            self.entity("a", "vulnerability", "info"),
            self.entity("b", "vulnerability", "major"),
            self.entity("a", "security_hotspot", vulnerability_probability="medium"),
            self.entity("b", "security_hotspot", vulnerability_probability="low")]
        self.assert_measurement(
            response, value="4", total="100", entities=expected_entities,
            landing_url="https://sonar/dashboard?id=id&branch=master")

    async def test_security_warnings_hotspots_only(self):
        """Test that only the security hotspots are returned."""
        self.sources["source_id"]["parameters"]["security_types"] = ["security_hotspot"]
        json = dict(
            paging=dict(total="2"),
            hotspots=[
                dict(key="a", message="a", component="a", vulnerabilityProbability="MEDIUM"),
                dict(key="b", message="b", component="b", vulnerabilityProbability="LOW")])
        metric = dict(type="security_warnings", addition="sum", sources=self.sources)
        response = await self.collect(metric, get_request_json_return_value=json)
        expected_entities = [
            self.entity("a", "security_hotspot", vulnerability_probability="medium"),
            self.entity("b", "security_hotspot", vulnerability_probability="low")]
        self.assert_measurement(
            response, value="2", total="100", entities=expected_entities,
            landing_url="https://sonar/security_hotspots?id=id&branch=master")

    async def test_security_warnings_vulnerabilities_only(self):
        """Test that only the vulnerabilities are returned."""
        self.sources["source_id"]["parameters"]["security_types"] = ["vulnerability"]
        json = dict(
            total="2",
            issues=[
                dict(key="a", message="a", component="a", severity="INFO", type="VULNERABILITY"),
                dict(key="b", message="b", component="b", severity="MAJOR", type="VULNERABILITY")])
        metric = dict(type="security_warnings", addition="sum", sources=self.sources)
        response = await self.collect(metric, get_request_json_return_value=json)
        expected_entities = [self.entity("a", "vulnerability", "info"), self.entity("b", "vulnerability", "major")]
        self.assert_measurement(
            response, value="2", total="100", entities=expected_entities, landing_url=self.issues_landing_url)


#!/usr/bin/env python
# -*- coding: utf-8 -*-
# File: output_unit.py
# Author: Qian Ge <geqian1001@gmail.com>

import numpy as np
import matplotlib.pyplot as plt

def sigmoid(x):
    x = np.reshape(x, (-1))
    return np.exp(x) / (1 + np.exp(x))

def d_logsigmoid(x):
    return sigmoid(-x)

def softmax(x):
    stabel_x = x - np.max(x)
    return np.exp(x) / np.sum(np.exp(x))

def d_logsoftmax(x, true_idx):
    d_list = np.exp(x) / np.sum(np.exp(x))
    d_list[true_idx] = np.exp(x[true_idx]) / np.sum(np.exp(x)) - 1
    return d_list

if __name__ == '__main__':
    x = np.float32(np.arange(-5.0, 5.0, 0.01))
    sigmoid_x = sigmoid(x)

    plt.figure()
    plt.plot(x, sigmoid(x), label='sigmoid')
    plt.plot(x, -np.log(sigmoid_x), label='loss function')
    plt.plot(x, d_logsigmoid(x), label='gradient')
    plt.title('correct label y = 1')
    plt.xlabel('x')
    plt.legend()

    plt.figure()
    plt.plot(x, 1 - sigmoid(x), label='sigmoid')
    plt.plot(x, -np.log(1 - sigmoid_x), label='loss function')
    plt.plot(x, d_logsigmoid(-x), label='gradient')
    plt.title('correct label y = 0')
    plt.xlabel('x')
    plt.legend()

    x = -5.0
    cnt = 0
    x_list = []
    d_list = []
    while True:
        x_list.append(x)
        cnt += 1
        if cnt == 6000:
            break
        d = d_logsigmoid(x)
        d_list.append(d)
        x = -0.1 * d + x

    plt.figure()
    plt.plot(x_list)
    plt.xlabel('step')
    plt.title('Find optimal x to minimize loss function when correct label y = 1')

    x = [1, 2, 3]
    cnt = 0
    x_list = []
    d_list = []
    while True:
        x_list.append(x)
        cnt += 1
        if cnt == 500:
            break
        d = d_logsoftmax(x, 0)
        d_list.append(d)
        x = -0.1 * d + x

    x_list = np.array(x_list)
    d_list = np.array(d_list)

    plt.figure()
    plt.plot(x_list[:, 0], label='true class')
    plt.plot(x_list[:, 1], label='wrong class')
    plt.plot(x_list[:, 2], label='wrong class')

    plt.legend()
    plt.title('Find optimal x to minimize loss function')

    plt.figure()
    plt.plot(d_list[:, 0], label='true class')
    plt.plot(d_list[:, 1], label='wrong class')
    plt.plot(d_list[:, 2], label='wrong class')
    plt.title('gradient')
    plt.legend()

    plt.show()


from sklearn import datasets, metrics
from sklearn.ensemble import RandomForestClassifier
from sklearn.model_selection import train_test_split
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns
import random

def select_features(features,num):
    top = features[:num].index.tolist()

    tmp = []
    for a in top:
        tmp.append( ' '.join(a.split(' ')[:2]))
    top = tmp

    return top

def visualize_feature_importance(features):
    # Generate a bar plot
    sns.barplot(x=features, y=features.index)

    # Add labels to the graph of features
    plt.xlabel('Importance Score of Features')
    plt.ylabel('Features')
    plt.title("Visualizing Features and Imp_scores")
    plt.show()

def sample_inputs_generator(num):
    sample = [[] for _ in range(random.randint(1,5))]
    for lst in sample:
        for _ in range(num):
            lst.append(round(random.uniform(0,9),1))

    return sample

if __name__=="__main__":
    iris = datasets.load_iris()

    data=pd.DataFrame({
    'sepal length':iris.data[:,0],
    'sepal width':iris.data[:,1],
    'petal length':iris.data[:,2],
    'petal width':iris.data[:,3],
    'species':iris.target
    })

    x = iris.data
    y = iris.target

    # 70% for training data, 30% for testing data
    x_train,x_test,y_train,y_test = train_test_split(x,y,test_size=0.3)

    # Generate and Train the RF model
    rfc = RandomForestClassifier(criterion = 'entropy', n_estimators = 8, n_jobs = -1)
    model = rfc.fit(x_train,y_train)

    #Calculate feature importance in descending order
    ft_importances = pd.Series(model.feature_importances_,index=iris.feature_names).sort_values(ascending=False)
    visualize_feature_importance(ft_importances)

    #Generate the datasets Again
    num = 3
    top = select_features(ft_importances,num)
    x = data[top]

    # Resetting training and testing data
    x_train,x_test,y_train,y_test = train_test_split(x,y,test_size=0.3)
    model = rfc.fit(x_train,y_train)

    # Calculate the accuracy score with test data
    y_pred = model.predict(x_test)
    print("Accuracy score:",metrics.accuracy_score(y_test, y_pred))
    print(metrics.classification_report(y_test, y_pred, target_names=iris.target_names))

    # predict with the sample data
    sample_inputs = sample_inputs_generator(num)
    sample_outputs = model.predict(sample_inputs).tolist()
    outs = []
    for o in sample_outputs:
        outs.append(iris.target_names[o])
    result = pd.DataFrame(sample_inputs,columns=top)
    result['species']=outs
    print("-"*100)
    print(result)


# /*
# * Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved.
# *
# * Permission is hereby granted, free of charge, to any person obtaining a copy of this
# * software and associated documentation files (the "Software"), to deal in the Software
# * without restriction, including without limitation the rights to use, copy, modify,
# * merge, publish, distribute, sublicense, and/or sell copies of the Software, and to
# * permit persons to whom the Software is furnished to do so.
# *
# * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
# * INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A
# * PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
# * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
# * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
# * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
# */

import json
import boto3
import botocore
import logging
from botocore.vendored import requests

logger = logging.getLogger()
logger.setLevel(logging.INFO)

sc = boto3.client('servicecatalog')

responsedata = {}

def lambda_handler(event, context):
    # Need to make sure we don't waste time if the request type is
    # update or delete.  Exit gracefully
    if event['RequestType'] == "Delete":
        logger.info(f'Request Type is Delete; unsupported')
        cfnsend(event, context, 'SUCCESS', responsedata)
        return event
    if event['RequestType'] == "Update":
        logger.info(f'Request Type is Update; unsupported')
        cfnsend(event, context, 'SUCCESS', responsedata)
        return event

    # Check if product provided
    if not 'ProductName' in event['ResourceProperties']:
        logger.info(f'Product Not Provided')
        responsedata['Id'] = 'Product not provided'
        cfnsend(event, context, 'FAILED', responsedata, 'Product name was not provided in lambda invocation')
        return event

    # get product name from CFN calling lambda
    productName = f'sc-{event["ResourceProperties"]["ProductName"]}-product'
    logger.info(f'Searching product: {productName}')
    # search product in AWS Service Catalog
    response = sc.search_products_as_admin(
        SortBy='Title',
        SortOrder='ASCENDING',
        ProductSource='ACCOUNT',
        Filters={
                'FullTextSearch': [
                    productName
                ]
            }
    )

    productList=[]
    #Iterate through all return products
    for p in response['ProductViewDetails']:
        # find the searching product
        if productName in p['ProductViewSummary']['Name']:
            productList.append({'ProductId' : p['ProductViewSummary']['ProductId'], 'CreatedTime' : p['CreatedTime'] })

    # if product found
    if productList:
        # if more the on product get the latest one based on the date/time
        last_product_Id = max(productList,key=lambda item:item['CreatedTime'])
        prodId = last_product_Id['ProductId']
        responsedata['ProductId'] = prodId

        # search artifacts (versions) of product
        prodArtif = sc.list_provisioning_artifacts(
            AcceptLanguage='en',
            ProductId=prodId
        )

        # if version provided in cfn
        if 'Version' in event['ResourceProperties']:
            logger.info(f'Searching version: {event["ResourceProperties"]["Version"]}')
            for a in prodArtif['ProvisioningArtifactDetails']:
                # find artifact that match provided version
                if a['Name'] == event['ResourceProperties']['Version']:
                    responsedata['ArtifactId'] = a['Id']

            if not 'ArtifactId' in responsedata:
                logger.info('Version not found')
                responsedata['ArtifactId'] = 'Version not found'
                cfnsend(event, context, 'FAILED', responsedata, 'Version not found in Service Catalog')
                return event
        # otherwise return last one based on the date/time
        else:
            last_artifact_Id = max(prodArtif['ProvisioningArtifactDetails'],key=lambda item:item['CreatedTime'])
            responsedata['ArtifactId'] = last_artifact_Id['Id']

        logger.info(f'Respond Product Id: {responsedata["ProductId"]}')
        logger.info(f'Respond Artifact Id: {responsedata["ArtifactId"]}')
        # Return product info back to CFN
        cfnsend(event, context, 'SUCCESS', responsedata)
    else:
        # if product not found failed CFN
        logger.info('Product Not Found')
        responsedata['Id']='Product not found'
        cfnsend(event, context, 'FAILED', responsedata, 'Product not found in Service Catalog')

    return event

def cfnsend(event, context, responseStatus, responseData, reason=None):
    if 'ResponseURL' in event:
        responseUrl = event['ResponseURL']
        # Build out the response json
        responseBody = {}
        responseBody['Status'] = responseStatus
        responseBody['Reason'] = reason or 'CWL Log Stream =' + context.log_stream_name
        responseBody['PhysicalResourceId'] = context.log_stream_name
        responseBody['StackId'] = event['StackId']
        responseBody['RequestId'] = event['RequestId']
        responseBody['LogicalResourceId'] = event['LogicalResourceId']
        responseBody['Data'] = responseData
        json_responseBody = json.dumps(responseBody)

        logger.info(f'Response body: + {json_responseBody}')

        headers = {
            'content-type': '',
            'content-length': str(len(json_responseBody))
        }
        # Send response back to CFN
        try:
            response = requests.put(responseUrl,
                                    data=json_responseBody,
                                    headers=headers)
            logger.info(f'Status code: {response.reason}')
        except Exception as e:
            logger.info(f'send(..) failed executing requests.put(..):  {str(e)}')


import stellar_sdk
import requests

server = stellar_sdk.Server()

SQKeypair = stellar_sdk.Keypair.from_secret(input("Stellar Quest Secret: "))
account = server.load_account(SQKeypair.public_key)

resp = requests.get(f"https://horizon-testnet.stellar.org/accounts/{SQKeypair.public_key}").json()

transaction = stellar_sdk.TransactionBuilder(source_account=account)

for key in resp["data"].keys():
    transaction.append_manage_data_op(
        key, None
    )

if transaction.operations:
    transaction = transaction.build()
    transaction.sign(SQKeypair)

    resp = server.submit_transaction(transaction)
    print(resp)
else:
    print("No data to remove")


# Run this app with `python app.py` and
# visit http://127.0.0.1:8050/ in your web browser.

import dash
import dash_core_components as dcc
import dash_html_components as html
import plotly.express as px
import pandas as pd

external_stylesheets = ['https://codepen.io/chriddyp/pen/bWLwgP.css']

app = dash.Dash(__name__, external_stylesheets=external_stylesheets)

df = pd.read_csv('https://gist.githubusercontent.com/chriddyp/5d1ea79569ed194d432e56108a04d188/raw/a9f9e8076b837d541398e999dcbac2b2826a81f8/gdp-life-exp-2007.csv')

fig = px.scatter(df, x="gdp per capita", y="life expectancy",
                 size="population", color="continent", hover_name="country",
                 log_x=True, size_max=60)

app.layout = html.Div([
    dcc.Graph(
        id='life-exp-vs-gdp',
        figure=fig
    )
])

if __name__ == '__main__':
    app.run_server(debug=True)


from sklearn.cluster import KMeans

import pickle
import pandas as pd

def run():

    df = pd.read_pickle('../Data/pickles/clustering_data')
    acc_num = df['accountNumber']
    df.drop(['accountNumber'], axis=1, inplace=True)
    # print(df)
    #
    # scores = {'clusters': [], 'scores': []}
    #
    # for num_of_clusters in range(1, 31):
    #     scores['clusters'].append(num_of_clusters)
    #     scores['scores'].append(KMeans(n_clusters=num_of_clusters, verbose=1, random_state=0).fit(df).score(df))
    #
    # with open('../Data/pickles/elbow', 'wb') as file:
    #     pickle.dump(scores, file)

    kmeans = KMeans(n_clusters=3, verbose=1, random_state=0)

    cluster_labels = [f'cluster_{i}' for i in kmeans.fit(df).labels_]

    df['cluster'] = cluster_labels
    df['accountNumber'] = acc_num
    df.to_pickle('../Data/pickles/clustered_df', protocol=4)

    print(df)

if __name__ == '__main__':
    run()


# -*- coding: utf-8 -*-
"""
Created on Tue Jun 12 11:13:39 2018
Functions:
1) Read all images from an image directory, specified by root,
2) Then resize the images one by one to a given size, specified by resize_size.

   scipy.misc.imread(/path/to/image/.image_extension)
   scipy.misc.imresize(image_to_resize, (resize_size, resize_size))
   scipy.misc.imsave(save_root + name, arr=img_data)

   Also you can refer to:  matplotlib.pyplot.imread, imresize, imsave

@author: Yongwei
"""

import os
from scipy.misc import imread, imresize, imsave

root = '/path/to/Dataset_original/'
save_root = '/path/to/Dataset_resized/'

resize_size = 128

# check directory
if not os.path.isdir(save_root):
    os.mkdir(save_root)

img_list = os.listdir(root)
# print(img_list) #img_list contains complete name of each image, e.g.,t1.bmp, t2.bmp...

for i in range(len(img_list)):
    img = imread(root + img_list[i])
    img = imresize(img, (resize_size, resize_size))

    imsave(save_root + img_list[i], arr=img)

    if (i+1) % 10 == 0:
        print ('%d images complete' % (i+1))


from django.urls import path
from . import views

app_name = "pages"
urlpatterns = [
    path('',views.home,name="home"),
    path('book/<int:book_id>/',views.detail,name="detail"),
    path('book/<int:book_id>/add/',views.addToCart,name="addToCart"),
    path('book/<int:book_id>/remove/',views.removeFromCart,name="removeFromCart"),
    path('cart/',views.cart,name="cart"),
    path('search/',views.searchview,name="search")
]


import requests
import pprint  # для печати в красивом формате

DOMAIN = 'https://api.hh.ru/'  # будем работать с api
url_vacancies = f'{DOMAIN}vacancies'  # адрес для  получения вакансий (из документации по api)

params = {                         # задаем параметры запроса
    #'text' : 'NAME: (Python OR Java)  AND COMPANY_NAME: (SBERBANK OR GAZPROM OR YANDEX) AND (DJANGO OR SPRING)',
    'text' : 'NAME: (Python OR Java) AND (DJANGO OR SPRING)',
    #'area': 95, # По городу Москва - 1,  54 - Красноярск, 66 - Нижний Ноывгород, 96 - Ижевск (24)
    # есть страницы, так как данных много
    'page' : 1     # выводим на страницу 1
}

result = requests.get(url_vacancies, params=params).json() # результат запроса помещаем в result в формате JSON
pprint.pprint(result) # печатаем все найденные записи
#print(result['items'][0]['alternate_url'])


import numpy as np
import torch
from pytorch_base_network import PyTorchBaseNetwork
from torch import nn
from torch import optim
import torch.nn.functional as F

import helper

import matplotlib.pyplot as plt
from torchvision import datasets, transforms

# Define a transform to normalize the data
transform = transforms.Compose([transforms.ToTensor(),
                              transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5)),
                             ])
# Download and load the training data
trainset = datasets.MNIST('MNIST_data/', download=True, train=True, transform=transform)
trainloader = torch.utils.data.DataLoader(trainset, batch_size=64, shuffle=True)
# Download and load the test data
testset = datasets.MNIST('MNIST_data/', download=True, train=False, transform=transform)
testloader = torch.utils.data.DataLoader(testset, batch_size=64, shuffle=True)

# Grab some data and show it
dataiter = iter(trainloader)
images, labels = dataiter.next()
plt.imshow(images[1].numpy().squeeze(), cmap='Greys_r');

# Flatten the data for it to be fed to the fully connected neural network
# data will be a 1D vector, new images shape is (batch size, color channels, image pixels)
images.resize_(64, 1, 784)
# or images.resize_(images.shape[0], 1, 784) to not automatically get batch size
# Forward pass through the network

#select the first image
img_idx = 0
img = images[img_idx, :]

"""

# build the default pytorch base network
model_def = PyTorchBaseNetwork()
#pass the first image trough
ps = model_def.forward(img)
#classify (before training, won't work)
helper.view_classify(img.view(1, 28, 28), ps)

#alternatively, a sequential network can be built in place:

model_seq = nn.Sequential(OrderedDict([
                      ('fc1', nn.Linear(784,200)),
                      ('relu1', nn.ReLU()),
                      ('fc2', nn.Linear(200, 100)),
                      ('relu2', nn.ReLU()),
                      ('output', nn.Linear(100,10)),
                      ('softmax', nn.Softmax(dim=1))]))

ps = model_seq.forward(img)
helper.view_classify(img.view(1, 28, 28), ps)

"""

#as asked in the exercise, build a different network with 4 layers (3 inner +output)
model=PyTorchBaseNetwork(784, None, [100,50],10)
#classify (before training, won't work)
from collections import OrderedDict
model_seq = nn.Sequential(OrderedDict([
                      ('fc1', nn.Linear(784,200)),
                      ('relu1', nn.ReLU()),
                      ('fc2', nn.Linear(200, 100)),
                      ('relu2', nn.ReLU()),
                      ('output', nn.Linear(100,10))]))

ps = model_seq.forward(img)
helper.view_classify(img.view(1, 28, 28), ps)

criterion = nn.CrossEntropyLoss()
optimizer = optim.SGD(model_seq.parameters(), lr=0.01)

print('Initial weights - ', model_seq.fc1.weight)

images, labels = next(iter(trainloader))
images.resize_(64, 784)

# Clear the gradients, do this because gradients are accumulated
optimizer.zero_grad()

# Forward pass, then backward pass, then update weights
output = model_seq.forward(images)
loss = criterion(output, labels)
loss.backward()
print('Gradient -', model_seq.fc1.weight.grad)
optimizer.step()
print('Updated weights - ', model_seq.fc1.weight)

# Define a transform to normalize the data
transform = transforms.Compose([transforms.ToTensor(),
                              transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5)),
                             ])
# Download and load the training data
trainset = datasets.MNIST('MNIST_data/', download=True, train=True, transform=transform)
trainloader = torch.utils.data.DataLoader(trainset, batch_size=64, shuffle=True)
# Download and load the test data
testset = datasets.MNIST('MNIST_data/', download=True, train=False, transform=transform)
testloader = torch.utils.data.DataLoader(testset, batch_size=64, shuffle=True)


from locust import HttpUser, task
from random import randint

class WebsiteUser(HttpUser):
    @task
    def view_movies(self):
        self.client.get("/api/movies", name="/api/movies")

    @task
    def view_movie(self):
        movie_id = randint(14, 27)
        self.client.get(f"/api/movies/{movie_id}", name="/api/movies/:id")


from typing import List

from fastapi import Body, HTTPException, Depends, APIRouter
from sqlalchemy.orm import Session

from database.connection import get_db
from database.orm import ToDo
from database.repository import create_todo, get_todos, get_todo_by_todo_id, update_todo, delete_todo
from module import httprequest
from schema.request import CreateToDoRequest, CrawlingRequest
from schema.response import ToDoSchema, ToDoListSchema

router = APIRouter()

@router.post("/todos", status_code=201)
def create_todo_handler(
        request: CreateToDoRequest,
        session: Session = Depends(get_db)
) -> ToDoSchema:
    todo: ToDo = ToDo.create(request=request)
    todo: ToDo = create_todo(session=session, todo=todo)
    return ToDoSchema.from_orm(todo)

@router.get("/todos", status_code=200)
def get_todos_handler(
        order: str | None = None,
        session: Session = Depends(get_db),
) -> ToDoListSchema:  # | None = None > 스웨거 Required 해제.
    todos: List[ToDo] = get_todos(session=session)
    if order and order == "DESC":
        return ToDoListSchema(
            todos=[ToDoSchema.from_orm(todo) for todo in todos[::-1]]
        )
    return ToDoListSchema(
        todos=[ToDoSchema.from_orm(todo) for todo in todos]
    )

@router.get("/todos/{todo_id}", status_code=200)
def get_todo_handler(
        todo_id: int,
        session: Session = Depends(get_db),
) -> ToDoSchema:
    todo: ToDo | None = get_todo_by_todo_id(session=session, todo_id=todo_id)
    if todo:
        return ToDoSchema.from_orm(todo)
    raise HTTPException(status_code=404, detail="ToDo Not Found")

@router.patch("/todos/{todo_id}", status_code=201)
def update_todo_handler(
        todo_id: int,
        is_done: bool = Body(..., embed=True),
        session: Session = Depends(get_db),
):
    todo: ToDo | None = get_todo_by_todo_id(session=session, todo_id=todo_id)
    if todo:
        todo.done if is_done else todo.undone()
        todo: ToDo = update_todo(session=session, todo=todo)
        return ToDoSchema.from_orm(todo)
    raise HTTPException(status_code=404, detail="ToDo Not Found")

@router.delete("/todos/{too_did}", status_code=204)
def delete_todo_handler(
        todo_id: int,
        session: Session = Depends(get_db),
):
    todo: ToDo | None = get_todo_by_todo_id(session=session, todo_id=todo_id)
    if not todo:
        raise HTTPException(status_code=404, detail="ToDo Not Found")

    delete_todo(session=session, todo_id=todo_id)


''' View functions for authorization and integration of apps in YA '''

# -*- coding: utf-8 -*-

import json
import traceback
import urllib
import requests
from django.http import HttpResponse
from django.shortcuts import render
import uuid

from django.urls import reverse
from django.contrib.auth.models import User
from django.http import HttpResponseRedirect
from django.shortcuts import render, redirect
from django.http import HttpResponse, HttpResponseBadRequest, HttpResponseServerError
from django.conf import settings
from django.views.decorators.csrf import csrf_exempt
from django.views.decorators.http import require_POST
from requests import Response
from yellowant import YellowAnt
from yellowant.messageformat import MessageClass, MessageAttachmentsClass, MessageButtonsClass, AttachmentFieldsClass

from ..records.commandcentre import CommandCentre
from requests.models import Response
from ..records.models import YellowUserToken,YellowAntRedirectState, AppRedirectState, DropBoxUserToken

# Create your views here.

def redirectToYellowAntAuthenticationPage(request):
    print("In redirectToYellowAntAuthenticationPage")
    '''Initiate the creation of a new user integration on YA
       YA uses oauth2 as its authorization framework.
       This method requests for an oauth2 code from YA to start creating a
       new user integration for this application on YA.
    '''

    # Generate a unique ID to identify the user when YA returns an oauth2 code
    user = User.objects.get(id=request.user.id)
    state = str(uuid.uuid4())

    # Save the relation between user and state so that we can identify the user
    # when YA returns the oauth2 code
    YellowAntRedirectState.objects.create(user=user.id, state=state)

    # Redirect the application user to the YA authentication page.
    # Note that we are passing state, this app's client id,
    # oauth response type as code, and the url to return the oauth2 code at.
    return HttpResponseRedirect("{}?state={}&client_id={}&response_type=code&redirect_url={}".format
                                (settings.YELLOWANT_OAUTH_URL, state, settings.YELLOWANT_CLIENT_ID,
                                 settings.YELLOWANT_REDIRECT_URL))

def yellowantRedirecturl(request):
    print("In yellowantRedirecturl")

    ''' Receive the oauth2 code from YA to generate a new user integration
        This method calls utilizes the YA Python SDK to create a new user integration on YA.
        This method only provides the code for creating a new user integration on YA.
        Beyond that, you might need to authenticate the user on
        the actual application (whose APIs this application will be calling) and store a relation
        between these user auth details and the YA user integration.
    '''

    # Oauth2 code from YA, passed as GET params in the url
    code = request.GET.get('code')

    # The unique string to identify the user for which we will create an integration
    state = request.GET.get("state")

    # Fetch user with help of state from database
    yellowant_redirect_state = YellowAntRedirectState.objects.get(state=state)
    user = yellowant_redirect_state.user

    # Initialize the YA SDK client with your application credentials
    y = YellowAnt(app_key=settings.YELLOWANT_CLIENT_ID,
                  app_secret=settings.YELLOWANT_CLIENT_SECRET,
                  access_token=None, redirect_uri=settings.YELLOWANT_REDIRECT_URL)

    # Getting the acccess token
    access_token_dict = y.get_access_token(code)
    print(access_token_dict)

    access_token = access_token_dict['access_token']

    # Getting YA user details
    yellowant_user = YellowAnt(access_token=access_token)
    profile = yellowant_user.get_user_profile()

    # Creating a new user integration for the application
    user_integration = yellowant_user.create_user_integration()
    hash_str = str(uuid.uuid4()).replace("-", "")[:25]
    ut = YellowUserToken.objects.create(user=user, yellowant_token=access_token,
                                        yellowant_id=profile['id'],
                                        yellowant_integration_invoke_name=user_integration\
                                            ["user_invoke_name"],
                                        yellowant_integration_id=user_integration\
                                            ['user_application'],
                                        webhook_id=hash_str)

    state = str(uuid.uuid4())
    AppRedirectState.objects.create(user_integration=ut, state=state)

    url = "https://www.dropbox.com/oauth2/authorize/"
    params = { 'response_type': 'code','client_id': settings.DROPBOX_CLIENT_ID,
               'redirect_uri': settings.DROPBOX_REDIRECT_URL,
               'state': state}
    url += '?' + urllib.parse.urlencode(params)
    return HttpResponseRedirect(url)

def dropboxRedirecturl(request):
    # DropBoxUserToken.objects.create(user_integration=ut)
    ''' OAuth2 at Dropbox server '''
    if request.method == "GET":
        print("In dropboxRedirecturl GET")

        code = request.GET.get("code", False)
        print(code)

        state = request.GET.get("state")
        print(state)

        dropbox_redirect_state = AppRedirectState.objects.get(state=state)
        ut = dropbox_redirect_state.user_integration
        error = request.GET.get('error', None)
        print(error)

        # Checking status of auth request
        if error == 'access_denied':
            return HttpResponse('Access denied')
        if state is None:
            return HttpResponseBadRequest()

        # Getting auth code
        auth_code = request.GET.get('code', None)
        if auth_code is None:
            return HttpResponseBadRequest()

        print("------ Auth code here -----")
        print (auth_code)

        ## Getting access token

        url = "https://api.dropboxapi.com/oauth2/token/"
        headers = {'Accept': 'application/json', 'content-type': 'application/x-www-form-urlencoded'}
        params = {'code': code, 'grant_type': 'authorization_code','client_id': settings.DROPBOX_CLIENT_ID,
                  'client_secret' : settings.DROPBOX_CLIENT_SECRET,'redirect_uri': settings.DROPBOX_REDIRECT_URL
                  }

        print("In POST request")
        r = requests.post(url, data=params, headers=headers)
        if r.status_code != 200:
            return HttpResponse(r.text)

        access_token_dict = json.loads(r.text)
        uid = access_token_dict['uid']
        accessToken = access_token_dict['access_token']
        account_id = access_token_dict['account_id']
        token_type = access_token_dict['token_type']

        DropBoxUserToken.objects.create(user_integration=ut,accessToken=accessToken,account_id=account_id)
        print(access_token_dict)

        url = settings.BASE_URL + "/webhook/" + ut.webhook_id + "/"
        print(url)

        return HttpResponseRedirect("/")

@csrf_exempt
def webhook(request, hash_str=""):

    '''Respond to the webhook verification (GET request) by echoing back the challenge parameter.'''
    print("Inside webhook")

    if request.method == "GET":
        print("Inside webhook validation")
        challenge = request.GET.get('challenge',None)

        if challenge != None:
            return HttpResponse(challenge,status=200)
        else:
            return HttpResponse(status=400)

    else:
        print("In notifications")
        webhook_id = hash_str
        data =  (request.body.decode('utf-8'))
        response_json = json.loads(data)
        print(response_json)

        data = {
	            "users": [],
	            "accounts": []
            }

        try:
            yellow_obj = YellowUserToken.objects.get(webhook_id=webhook_id)
            print(yellow_obj)
            access_token = yellow_obj.yellowant_token
            print(access_token)
            integration_id = yellow_obj.yellowant_integration_id
            service_application = str(integration_id)
            print(service_application)

            # Creating message object for webhook message

            webhook_message = MessageClass()
            webhook_message.message_text = "File/Folder updated !"
            attachment = MessageAttachmentsClass()
            attachment.title = "Updated file/folder details :"

            for i in range(0,len(response_json['list_folder']['accounts'])):
                field1 = AttachmentFieldsClass()
                field1.title = "Id : "
                field1.value = response_json['list_folder']['accounts'][i]
                data["accounts"].append(response_json['list_folder']['accounts'][i])
                attachment.attach_field(field1)

            attachment2 = MessageAttachmentsClass()
            attachment2.title = "User update details :"

            for i in range(0, len(response_json['delta']['users'])):
                field2 = AttachmentFieldsClass()
                field2.title = "Id : "
                field2.value = response_json['delta']['users'][i]
                data["users"].append(response_json['delta']['users'][i])
                attachment2.attach_field(field2)

            button = MessageButtonsClass()
            button.name = "1"
            button.value = "1"
            button.text = "Get all files and folders "
            button.command = {
                "service_application": service_application,
                "function_name": 'get_all_folders',
                "data" : {"path": "",
                "recursive": True,
                "include_media_info": False,
                "include_deleted": False,
                "include_has_explicit_shared_members": False,
                "include_mounted_folders": True}
                }

            attachment.attach_button(button)
            webhook_message.attach(attachment)
            webhook_message.attach(attachment2)
            #print(integration_id)
            print("-----------")
            print(data)
            print("------------")
            webhook_message.data = data
            # Creating yellowant object
            yellowant_user_integration_object = YellowAnt(access_token=access_token)

            # Sending webhook message to user
            send_message = yellowant_user_integration_object.create_webhook_message(
                requester_application=integration_id,
                webhook_name="files_folders_update", **webhook_message.get_dict())

            return HttpResponse("OK", status=200)

        except YellowUserToken.DoesNotExist:
            return HttpResponse("Not Authorized", status=403)

# def update_invoice(request,webhook_id):
#
#     """
#     Webhook function to notify user about newly created invoice
#     """
#     print("In update_invoice")
#
#     # Extracting necessary data
#     code = request.GET.get("code", False)
#
#     if code == request.codes.ok:
#         data = (request.body.decode('utf-8'))
#         response_json = json.loads(data)
#
#         try:
#             yellow_obj = YellowUserToken.objects.get(webhook_id=webhook_id)
#             print(yellow_obj)
#             access_token = yellow_obj.yellowant_token
#             print(access_token)
#             integration_id = yellow_obj.yellowant_integration_id
#             service_application = str(integration_id)
#             print(service_application)
#
#
#             # Creating message object for webhook message
#             webhook_message = MessageClass()
#             id = str(response_json['eventNotifications'][0]['dataChangeEvent']['entities'][0]['id'])
#             webhook_message.message_text = "Invoice " + str(response_json['eventNotifications'][0]['dataChangeEvent']['entities'][0]['id']) + " updated"
#             attachment = MessageAttachmentsClass()
#             attachment.title = "Invoice operations"
#
#             button_get_incidents = MessageButtonsClass()
#             button_get_incidents.name = "1"
#             button_get_incidents.value = "1"
#             button_get_incidents.text = "Get invoice details"
#             button_get_incidents.command = {
#                 "service_application": service_application,
#                 "function_name": 'read_invoice',
#                 "data": {"invoice_id":id}
#             }
#
#             attachment.attach_button(button_get_incidents)
#             webhook_message.attach(attachment)
#             #print(integration_id)
#
#             webhook_message.data = {"invoice_id":id}
#             # Creating yellowant object
#             yellowant_user_integration_object = YellowAnt(access_token=access_token)
#
#             # Sending webhook message to user
#             send_message = yellowant_user_integration_object.create_webhook_message(
#                 requester_application=integration_id,
#                 webhook_name="invoice_update", **webhook_message.get_dict())
#
#             return HttpResponse("OK", status=200)
#
#         except YellowUserToken.DoesNotExist:
#             return HttpResponse("Not Authorized", status=403)
#     else:
#         return HttpResponse(status=400)
#
#
# def add_new_invoice(request,webhook_id):
#
#     """
#     Webhook function to notify user about newly created invoice
#     """
#     print("In add_new_invoice")
#
#
#     # Extracting necessary data
#     code = request.GET.get("code", False)
#
#     if code == request.codes.ok:
#         data = (request.body.decode('utf-8'))
#         response_json = json.loads(data)
#
#         try:
#             yellow_obj = YellowUserToken.objects.get(webhook_id=webhook_id)
#             print(yellow_obj)
#             access_token = yellow_obj.yellowant_token
#             print(access_token)
#             integration_id = yellow_obj.yellowant_integration_id
#             service_application = str(integration_id)
#             print(service_application)
#
#             # Creating message object for webhook message
#             webhook_message = MessageClass()
#             id = str(response_json['eventNotifications'][0]['dataChangeEvent']['entities'][0]['id'])
#             webhook_message.message_text = "New invoice added\n" + "ID :" + str(response_json['eventNotifications'][0]['dataChangeEvent']['entities'][0]['id'])
#             attachment = MessageAttachmentsClass()
#             attachment.title = "Invoice operations"
#
#             button_get_incidents = MessageButtonsClass()
#             button_get_incidents.name = "1"
#             button_get_incidents.value = "1"
#             button_get_incidents.text = "Read invoice"
#             button_get_incidents.command = {
#                 "service_application": service_application,
#                 "function_name": 'read_invoice',
#                 "data": {"invoice_id": id}
#             }
#
#
#             webhook_message.data = {"invoice_id":id}
#             attachment.attach_button(button_get_incidents)
#             webhook_message.attach(attachment)
#             #print(integration_id)
#
#             # Creating yellowant object
#             yellowant_user_integration_object = YellowAnt(access_token=access_token)
#
#             # Sending webhook message to user
#             send_message = yellowant_user_integration_object.create_webhook_message(
#                 requester_application=integration_id,
#                 webhook_name="new_invoice", **webhook_message.get_dict())
#
#             return HttpResponse("OK", status=200)
#
#         except YellowUserToken.DoesNotExist:
#             return HttpResponse("Not Authorized", status=403)
#     else:
#         return HttpResponse(status=400)
#
# def add_new_customer(request,webhook_id):
#
#     """
#     Webhook function to notify user about newly created customer
#     """
#     print("In add_new_customer")
#     print(webhook_id)
#     # Extracting necessary data
#
#     code = request.GET.get("code", False)
#
#     if code == request.codes.ok:
#         data = (request.body.decode('utf-8'))
#         response_json = json.loads(data)
#         print(response_json)
#
#         try:
#             yellow_obj = YellowUserToken.objects.get(webhook_id=webhook_id)
#             print(yellow_obj)
#             access_token = yellow_obj.yellowant_token
#             print(access_token)
#             integration_id = yellow_obj.yellowant_integration_id
#             #DropBoxUserToken.objects.get(realmId=response_json['eventNotifications'][0]['realmId']).user_integration_id
#             service_application = str(integration_id)
#             print(service_application)
#
#             # Creating message object for webhook message
#             webhook_message = MessageClass()
#             id = str(response_json['eventNotifications'][0]['dataChangeEvent']['entities'][0]['id'])
#             webhook_message.message_text = "New customer added\n" + "ID :" + str(response_json['eventNotifications'][0]['dataChangeEvent']['entities'][0]['id'])
#             attachment = MessageAttachmentsClass()
#             attachment.title = "Get all customer details"
#
#             button_get_incidents = MessageButtonsClass()
#             button_get_incidents.name = "1"
#             button_get_incidents.value = "1"
#             button_get_incidents.text = "Get customer details"
#             button_get_incidents.command = {
#                 "service_application": service_application,
#                 "function_name": 'get_customer_details',
#                 "data": {"customer_id": id}
#             }
#
#             attachment.attach_button(button_get_incidents)
#
#             webhook_message.data = {"customer_id":id}
#             webhook_message.attach(attachment)
#             #print(integration_id)
#
#             # Creating yellowant object
#             yellowant_user_integration_object = YellowAnt(access_token=access_token)
#
#             # Sending webhook message to user
#             send_message = yellowant_user_integration_object.create_webhook_message(
#                 requester_application=integration_id,
#                 webhook_name="new_customer", **webhook_message.get_dict())
#             return HttpResponse("OK", status=200)
#
#         except YellowUserToken.DoesNotExist:
#             return HttpResponse("Not Authorized", status=403)
#     else:
#         return HttpResponse(status=400)
#

@csrf_exempt
def yellowantapi(request):
    try:
        """
        Recieve user commands from YA
        """

        # Extracting the necessary data
        data = json.loads(request.POST['data'])
        args = data["args"]
        service_application = data["application"]
        verification_token = data['verification_token']
        function_name = data['function_name']
        # print(data)

        # Verifying whether the request is actually from YA using verification token
        if verification_token == settings.YELLOWANT_VERIFICATION_TOKEN:

        # Processing command in some class Command and sending a Message Object
            message = CommandCentre(data["user"], service_application, function_name, args).parse()
        # Appropriate function calls for corresponding webhook functions
            # Figure out

        # Returning message response
            return HttpResponse(message)
        else:
            # Handling incorrect verification token
            error_message = {"message_text": "Incorrect Verification token"}
            return HttpResponse(json.dumps(error_message), content_type="application/json")

    except Exception as e:
        # Handling exception
        print(str(e))
        traceback.print_exc()
        return HttpResponse("Something went wrong !")


import pbr.version

version_info = pbr.version.VersionInfo('python_secrets')
try:
    __version__ = version_info.version_string()
except AttributeError:
    __version__ = '18.11.5'

__author__ = 'Dave Dittrich'
__email__ = 'dave.dittrich@gmail.com'

__all__ = ['__author__', '__email__', '__version__']

# vim: set fileencoding=utf-8 ts=4 sw=4 tw=0 et :


import frappe,requests,os
from frappe.model.document import Document

class SyncEHRBase(Document):
	def on_submit(self):
		data = {
   		"q": 'select e/ehr_id/value as ehrId, c from EHR e contains COMPOSITION c[openEHR-EHR-COMPOSITION.final_demographics.v0]',
			"resources": [resource.resource for resource in self.resources]
		}
		url = "http://openhim-core.lafia.io:5001/v1"
		print(data)
		response = requests.post(url, json=data, verify=False)

		print(response)


from typing import List

class Solution:
    def findRadius(self, houses: List[int], heaters: List[int]) -> int:

        """存放每个房屋与加热器的最短距离"""
        res = []
        heaters.sort()
        for c in houses:
            """二分查找"""
            left = 0
            right = len(heaters) - 1
            while left < right:
                mid = (left + right) >> 1
                if heaters[mid] < c:
                    left = mid + 1
                else:
                    right = mid
            """得出的 left 所指的加热器并不一定是离房屋 c 最近的一个，需要判断以下情况"""
            if heaters[left] == c:
                """若找到的值等于 c ，则说明 c 房屋处放有一个加热器，c 房屋到加热器的最短距离为 0"""
                res.append(0)

            elif heaters[left] < c:
                """若该加热器的坐标值小于 c ，说明该加热器的坐标与 c 之间没有别的加热器"""
                res.append(c - heaters[left])

            elif left == 0:
                """
                若left == 0 即二分查找的结果指向第一个加热器的坐标,说明 heaters[left] 坐标的房屋之前的位置
                未放置加热器,直接相减就是到房屋 c 最近加热器的距离
                """
                res.append(heaters[left] - c)

            else:
                """
                若left不等于 0 ，说明 c 介于left和left-1之间，房屋到加热器的最短距离就是left和left - 1处
                加热器与 c 差值的最小值.
                """
                res.append(min(heaters[left] - c, c - heaters[left - 1]))

        return max(res)


from DB_Handle import get_specified_shop
from input_check_code import readDianPingCookies,readDianPingCookies_for_allComments
from decode_encrypted_stream import css_get_for_shopDetails,decode_for_input_encrypted_stream,css_decode,css_decode_shoplist
from pyquery import PyQuery as pq
import time
import jieba
from input_check_code import readDianPingCookies
from shop_info_Commercial_area_specify_catagory_max_page import get_brower_driver
from dianping_check_dir import get_analyze_whole_dir

from matplotlib import pyplot as plt

from wordcloud import WordCloud

from PIL import Image
import random
import numpy as np
import sys
import os

def get_pinglun_text_from_file():
    analyze_dir = get_analyze_whole_dir()
    filename = analyze_dir + 'pinglun.txt'
    pinglun_all_list = []
    with open(filename,encoding="utf-8") as f:
        for line in f.readlines():
            pinglun_all_list.append(line)
    print('pinglun len: ',len(pinglun_all_list))

def get_all_pinglun_for_specified_food(shop_info):
    analyze_dir = get_analyze_whole_dir()
    filename = analyze_dir + 'pinglun.txt'
    fp = open(filename, 'w')
    fp.close()
    shop_url = shop_info[4]
    driver = get_brower_driver()
    #shop_url_tmp = 'http://www.dianping.com/shop/112133141'
    driver.get(shop_url)
    time.sleep(10)
    driver.find_element_by_css_selector('.comment-all a').click()
    time.sleep(1)

    tbCookies = readDianPingCookies_for_allComments()
    for cookie in tbCookies:
        print('cookie,tbCookies[cookie]: ', cookie, tbCookies[cookie])
        driver.add_cookie({"domain": "www.dianping.com",
                           "name": cookie,
                           "value": tbCookies[cookie],
                           "path": '/',
                           "expires": None})

    shop_url_tmp = shop_url + '/review_all'
    driver.get(shop_url_tmp)
    time.sleep(1)
    page_content = driver.page_source
    doc = pq(page_content)

    dict_svg_text={}
    list_svg_y=[]
    dict_css_x_y={}
    pinglun_all_list = []
    love_Food_List = []
    dict_svg_text, list_svg_y, dict_css_x_y,shoplist_or_eachShop= css_get_for_shopDetails(doc)
    cur_page = doc('.reviews-pages .PageSel').text()
    max_page = 1
    print('cur_page: ',cur_page)
    for each in doc('.reviews-pages .PageLink'):
        max_page = pq(each).text()
        print(pq(each).text())

    while int(cur_page) <= int(max_page):

        pinglunLi = doc("div.reviews-items > ul > li").items()
        pinglun_index = 1
        for data in pinglunLi:
            #print(each)
            # 用户名
            print('cur_page,max_page,pinglun_index: ', cur_page, max_page,pinglun_index)
            userName = data("div.main-review > div.dper-info > a").text()
            # 用户ID链接
            if data("div.main-review > div.dper-info > a").text() != '匿名用户':
                userID = "http://www.dianping.com" + data("div.main-review > div.dper-info > a").attr("href")
            # 用户评分星级[10-50]
            startShop = str(data("div.review-rank > span").attr("class")).split(" ")[1].replace("sml-str", "")
            # 用户描述：机器：非常好 环境：非常好 服务：非常好 人均：0元
            describeShop = data("div.review-rank > span.score").text()
            print('describeShop: ', describeShop)
            # 关键部分，评论HTML,待处理，评论包含隐藏部分和直接展示部分，默认从隐藏部分获取数据，没有则取默认部分。（查看更多）
            pinglun = data("div.review-words.Hide").html()
            try:
                len(pinglun)
            except:
                pinglun = data("div.review-words").html()
            # 该用户喜欢的美食
            loveFood = data("div.main-review > div.review-recommend").text()
            # 发表评论的时间
            pinglunTime = data("div.main-review > div.misc-info.clearfix > span.time").text()
            print('old pinglun: ',pinglun)
            transfer_pinglun = css_decode_shoplist(dict_css_x_y, dict_svg_text, list_svg_y, pinglun)
            with open(filename, 'a+', encoding="utf-8") as f:
                f.write(transfer_pinglun + "\n")
            print("userName:", userName)
            print("userID:", userID)
            print("startShop:", startShop)
            print("describeShop:", describeShop)
            print("loveFood:", loveFood)
            print("pinglunTime:", pinglunTime)
            #print('before pinglun decode: ', pinglun)
            print("pinglun:", transfer_pinglun)
            pinglun_all_list.append(transfer_pinglun)
            print("*" * 100)
            pinglun_index = pinglun_index + 1
            if loveFood != '':
                love_Food_List.append(loveFood)

        if int(max_page) > 1 and int(cur_page) == int(max_page):
            break
        if int(cur_page) < int(max_page):
            driver.find_element_by_css_selector('.reviews-pages .NextPage').click()
            time.sleep(random.randint(3, 5))
            page_content = driver.page_source
            doc = pq(page_content)
            cur_page = doc('.reviews-pages .PageSel').text()
    print('pinglun_all_list: ',len(pinglun_all_list))

    return pinglun_all_list
#
#
##
def generate_wordCloud(pinglun_all_list):
    font = r'D:\task_timer_exam\指定距离指定品类\can_yin\code\ttf字体集合\MSYH.TTF'#设置字体
    #
    text = ''
    counter = 0
    for each in pinglun_all_list:
        print(each)
        print('-' * 100)
        counter = counter + 1
        print('counter:  ',counter + 1)
        text = text + each + '\n'
    #text = (open(r"C:\Users\ToOmMyY\Desktop\土味.txt")).read()
    #
    cut = jieba.cut(text)# 分词

    string =' '.join(cut)

    print(len(string))

    img = Image.open(r"D:\task_timer_exam\指定距离指定品类\can_yin\code\wordCloud.png")# 打开图片

    img_array = np.array(img)# 将图片装换为数组

    stopword = ['xa0']# 设置停止词，也就是你不想显示的词，这里这个词是我前期处理没处理好，你可以删掉他看看他的作用

    wc = WordCloud(

    background_color='white',#设置背景

    width=2000,#设宽

    height=1000,#设高

    mask=img_array,
    max_words=500,
    font_path=font,

    stopwords=stopword

    )

    wc.generate_from_text(string)# 绘制图片

    plt.imshow(wc)

    plt.axis('off')

    plt.figure()

    #plt.show()# 显示图片

    wc.to_file(r"D:\task_timer_exam\指定距离指定品类\can_yin\code\呼玛路xxx.png")# 保存图片

if __name__ == '__main__':
    shop_name = '那都不是锅'
    shop_addr = '呼玛路'
    shop_info = get_specified_shop(shop_name, shop_addr)
    print(shop_info)
    pinglun_all_list = get_all_pinglun_for_specified_food(shop_info)
    generate_wordCloud(pinglun_all_list)

    #font = r'D:\task_timer_exam\指定距离指定品类\can_yin\code\ttf字体集合\MSYH.TTF'  # 设置字体
    #text = '中了霸王餐，果然名不虚传，FM天天听到这家店广告，今天来体验下，先喝两碗汤，非常浓稠，再加汤就不好喝了，一定要先喝，海鲜和有机蔬菜都新鲜，非常干净，港式火锅第一次吃，感受不错'
    #cut = jieba.cut(text)
    #string = ' '.join(cut)
    #img = Image.open(r"D:\task_timer_exam\指定距离指定品类\can_yin\code\wordCloud.png")  # 打开图片
    #img_array = np.array(img)
    #stopword = ['xa0']  # 设置停止词，也就是你不想显示的词，这里这个词是我前期处理没处理好，你可以删掉他看看他的作用
    #wc = WordCloud(
    #    background_color='white',  # 设置背景
    #    width=2000,  # 设宽
    #    height=1000,  # 设高
    #    mask=img_array,
    #    font_path=font,
    #    stopwords=stopword
    #)
    #wc.generate_from_text(string)  # 绘制图片
    #plt.imshow(wc)
    #plt.axis('off')
    #plt.figure()
    ##plt.show()  # 显示图片
    #wc.to_file(r"D:\task_timer_exam\指定距离指定品类\can_yin\code\呼玛路xxx.png")  # 保存图片


from os import listdir
import pandas as pd
import re

from tools import marks_google_dir, str_to_tuple
from gps_tools import haversine_distance

def label_outls(outls: pd.DataFrame, series_name: str, radius=10) -> list[int]:
    '''
        Asign a label to every outlier by searching the nearest mark using
        haversine distance.

        Parameters
        --------------

        outls: The set of outliers.
        series_name: The name of the series to choose the corresponding mark set.
        radius: The radius (in meters) within where to search for a mark to
        assign the label.

        Returns
        ---------------

        A list of ints representing the classes

    '''

    marks = find_marks_file(series_name)

    classes = [0]*len(outls)

    if not marks is None:
        for outl_idx in range(len(outls)):
            for mark_idx in range(len(marks)):
                mdf_location_index = marks.columns.get_loc("Location")
                mark_location = str_to_tuple(str(marks.iloc[mark_idx, mdf_location_index]))
                odf_latitude_index = outls.columns.get_loc("Latitude")
                odf_longitude_index = outls.columns.get_loc("Longitude")
                outl_location = (float(str(outls.iloc[outl_idx, odf_latitude_index])), float(str(outls.iloc[outl_idx, odf_longitude_index])))

                distance = haversine_distance(mark_location, outl_location, True)

                if distance < radius:
                    classes[outl_idx] = 1
                    break

    return classes

def find_marks_file(series_name: str) -> pd.DataFrame | None:
    '''
        Uses the time series identifier name to find the corresponding
        marks file.

        Parameters
        --------------

        series_name: Name of the time series.

        Returns
        --------------

        A pandas dataframe with the corresponding marks.

    '''

    marks_regex = rf"{series_name}_marks"

    for f_name in listdir(marks_google_dir):
        if re.match(marks_regex, f_name):
            return pd.read_csv(f"{marks_google_dir}/{marks_regex}.csv")

    return None


import os
import dotenv
from celery import Celery
from celery.schedules import crontab
from celery.signals import setup_logging

@setup_logging.connect
def void(*args, **kwargs):
    """Override celery's logging setup to prevent it from altering our settings.
    github.com/celery/celery/issues/1867

    :return void:
    """
    pass

dotenv.load_dotenv()
os.environ.setdefault("DJANGO_SETTINGS_MODULE", "macantine.settings")

app = Celery("macantine", broker=os.getenv("REDIS_URL"), backend=os.getenv("REDIS_URL"), include=["macantine.tasks"])
app.worker_hijack_root_logger = False

# At 10:00 on every day from Monday through Friday.
daily_week = crontab(hour=10, minute=0, day_of_week="1-5")
nightly = crontab(hour=4, minute=0, day_of_week="*")
weekly = crontab(hour=4, minute=0, day_of_week=6)
every_minute = crontab(minute="*/1")  # For testing purposes

app.conf.beat_schedule = {
    "no_canteen_first_reminder": {
        "task": "macantine.tasks.no_canteen_first_reminder",
        "schedule": daily_week,
    },
    "no_canteen_second_reminder": {
        "task": "macantine.tasks.no_canteen_second_reminder",
        "schedule": daily_week,
    },
    "no_diagnostic_first_reminder": {
        "task": "macantine.tasks.no_diagnostic_first_reminder",
        "schedule": daily_week,
    },
    "fill_missing_geolocation_data": {
        "task": "macantine.tasks.fill_missing_geolocation_data",
        "schedule": nightly,
    },
    "delete_old_historical_records": {
        "task": "macantine.tasks.delete_old_historical_records",
        "schedule": nightly,
    },
    "export_datasets": {
        "task": "macantine.tasks.export_datasets",
        "schedule": weekly,
    },
}

app.conf.timezone = "Europe/Paris"

if __name__ == "__main__":
    app.start()