# Gabriella Miles, Farscope PhD Student, Bristol Robotics Laboratory
# Date created: 13th July 2020

import tensorflow.keras as keras
import tensorflow as tf
from keras.models import Model
from keras.layers import Dense, Input, Flatten, Dropout, LSTM
from datetime import datetime

class Classifier:
	def __init__(self, input_shape, num_classes, output_directory, variable, verbose=True, build=True):
		self.output_directory = output_directory
		self.verbose = verbose
		self.variable = variable

		if build:
			self.model = self.build_model(input_shape, num_classes)
			return

	def build_model(self, input_shape, num_classes):
		reverse_sequences = True

		input_layer = Input(input_shape)

		lstm_1 = LSTM(100, return_sequences=True, activation='relu', go_backwards=True)(input_layer)
		lstm_2 = LSTM(100, return_sequences=True, activation='relu')(lstm_1)
		lstm_3 = LSTM(100, activation='relu')(lstm_2)
		output_layer = Dense(num_classes, activation='softmax')(lstm_3)

		model = Model(inputs=input_layer, outputs = output_layer)
		model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])

		return model

	def fit(self, trainX, trainy, testX, testy, variable):
		# hyperparameters
		mini_bs = 16
		num_epochs = 5
		suffix = datetime.now().strftime("%Y%,%d_%H%M%S")
		log = "./logs/scalars/"+suffix
		my_callbacks = [
			tf.keras.callbacks.TensorBoard(log_dir=log),
			tf.keras.callbacks.TerminateOnNaN(),
			tf.keras.callbacks.CSVLogger("./logs/test_"+suffix+".csv", separator=",", append=False)
		]

		hist = self.model.fit(trainX, trainy, validation_data=(testX, testy), callbacks=my_callbacks, batch_size=mini_bs, epochs=num_epochs, verbose=self.verbose)

		_, accuracy = self.model.evaluate(testX, testy, batch_size=mini_bs, verbose=self.verbose)

		return accuracy


import json
import time

from datamanage.pro import exceptions as dm_pro_errors
from datamanage.pro.dataquality.mixins.base_mixins import BaseMixin
from datamanage.pro.dataquality.models.audit import (
    DataQualityAuditFunction,
    DataQualityAuditRule,
    DataQualityAuditRuleEvent,
    DataQualityAuditRuleTemplate,
    DataQualityAuditTask,
    DataQualityEvent,
    DataQualityMetric,
)
from datamanage.pro.dataquality.serializers.audit import (
    DataQualityAuditFunctionSerializer,
    DataQualityAuditTaskSerializer,
    DataQualityRuleMetricsSerializer,
    DataQualityRuleSerializer,
    DataQualityRuleTemplateSerializer,
)
from datamanage.pro.dataquality.serializers.base import DataQualityDataSetSerializer
from datamanage.pro.pizza_settings import RULE_AUDIT_TASK_QUEUE
from datamanage.utils.dbtools.redis_util import redis_lpush
from datamanage.utils.drf import DataPageNumberPagination
from django.db import transaction
from django_filters.rest_framework import DjangoFilterBackend
from rest_framework.response import Response

from common.decorators import detail_route, list_route, params_valid
from common.local import get_request_username
from common.log import logger
from common.views import APIModelViewSet, APIViewSet

class DataQualityRuleViewSet(BaseMixin, APIViewSet):
    lookup_field = "rule_id"

    @params_valid(serializer=DataQualityRuleSerializer)
    def create(self, request, params):
        """
        @api {post} /datamanage/dataquality/rules/ 创建数据质量审核规则

        @apiVersion 3.5.0
        @apiGroup DataQualityRule
        @apiName dataquality_audit_rules_create
        @apiDescription 创建数据质量审核规则

        @apiParam {String} data_set_id 数据集ID
        @apiParam {Number} bk_biz_id 业务ID
        @apiParam {String} rule_name 规则名称
        @apiParam {String} rule_description 规则描述
        @apiParam {Number} rule_template_id 规则模板ID
        @apiParam {Object} rule_config 规则配置
        @apiParam {String} rule_config_alias 规则配置别名
        @apiParam {String} event_name 事件名称
        @apiParam {String} event_alias 事件别名
        @apiParam {String} event_description 事件描述
        @apiParam {String} event_type 事件类型
        @apiParam {String} event_polarity 事件极性
        @apiParam {Number} event_currency 事件时效性
        @apiParam {String} sensitivity 敏感程度
        @apiParam {String} event_detail_template 事件详情模板
        @apiParam {List} notify_ways 通知方式
        @apiParam {List} receivers 接收人
        @apiParam {Object} convergence_config 收敛策略
        @apiParam {Number} convergence_config.duration 触发时间周期范围
        @apiParam {Number} convergence_config.alert_threshold 周期内触发次数
        @apiParam {Number} convergence_config.mask_time 屏蔽时间

        @apiParamExample {json} 参数样例:
        {
            "data_set_id": "591_test",
            "rule_name": "规则一",
            "rule_description": "规则一的描述",
            "rule_template_id": 1,
            "rule_config": [
                {
                    "metric": {
                        "metric_type": "data_flow",
                        "metric_name": "output_count"
                    },
                    "function": "greater",
                    "constant": {
                        "constant_type": "float",
                        "constant_value": 100
                    },
                    "operation": "and"
                },
                {
                    "metric": {
                        "metric_type": "data_profiling",
                        "metric_name": "null_rate",
                        "metric_field": "field1"
                    },
                    "function": "less",
                    "constant": {
                        "constant_type": "float",
                        "constant_value": 100
                    },
                    "operation": null
                }
            ],
            "rule_config_alias": "(输出量 > 100) & (field1 空值率 < 100)",
            "event_name": "rule1_event",
            "event_alias": "规则一的事件",
            "event_description": "规则一事件的描述",
            "event_type": "data_quality",
            "event_sub_type": "test",
            "event_polarity": "positive",
            "event_currency": 30,
            "sensitivity": "private",
            "event_detail_template": "test",
            "notify_ways": ["wechart"],
            "receivers": ["admin"],
            "convergence_config": {
                "duration": 1,
                "alert_threshold": 1,
                "mask_time": 60
            }
        }

        @apiSuccessExample Success-Response:
            HTTP/1.1 200 OK
            {
                "errors": {},
                "message": "ok",
                "code": "1500200",
                "result": true,
                "data": true
            }

        """
        bk_username = get_request_username()
        with transaction.atomic(using="bkdata_basic"):
            rule = DataQualityAuditRule.objects.create(
                data_set_id=params["data_set_id"],
                bk_biz_id=params["bk_biz_id"],
                rule_name=params["rule_name"],
                rule_template_id=params["rule_template_id"],
                rule_config=json.dumps(params["rule_config"]),
                rule_config_alias=params["rule_config_alias"],
                created_by=bk_username,
                generate_type="user",
                description=params["rule_description"],
            )

            event = DataQualityEvent.objects.create(
                event_id="{}_{}".format(params["bk_biz_id"], params["event_name"]),
                event_name=params["event_name"],
                event_alias=params["event_alias"],
                event_currency=params["event_currency"],
                event_type=params["event_type"],
                event_sub_type=params.get("event_sub_type"),
                event_polarity=params["event_polarity"],
                event_detail_template=params["event_detail_template"],
                sensitivity=params["sensitivity"],
                generate_type="user",
                created_by=bk_username,
                description=params["event_description"],
            )

            DataQualityAuditRuleEvent.objects.create(
                rule=rule,
                event=event,
                notify_ways=",".join(params["notify_ways"]),
                receivers=",".join(params["receivers"]),
                convergence_config=json.dumps(params["convergence_config"]),
                created_by=bk_username,
            )
        return Response(True)

    @params_valid(serializer=DataQualityRuleSerializer)
    def update(self, request, rule_id, params):
        """
        @api {put} /datamanage/dataquality/rules/{rule_id}/ 更新数据质量审核规则

        @apiVersion 3.5.0
        @apiGroup DataQualityRule
        @apiName dataquality_audit_rules_update
        @apiDescription 更新数据质量审核规则

        @apiParam {String} rule_name 规则名称
        @apiParam {String} rule_description 规则描述
        @apiParam {Number} rule_template_id 规则模板ID
        @apiParam {Object} rule_config 规则配置
        @apiParam {String} rule_config_alias 规则配置别名
        @apiParam {String} event_name 事件名称
        @apiParam {String} event_alias 事件别名（新增）
        @apiParam {String} event_description 事件描述
        @apiParam {String} event_type 事件类型
        @apiParam {String} event_sub_type 事件子类型（新增）
        @apiParam {String} event_polarity 事件极性
        @apiParam {Number} event_currency 事件时效性
        @apiParam {String} sensitivity 敏感程度
        @apiParam {String} event_detail_template 事件详情模板
        @apiParam {List} notify_ways 通知方式
        @apiParam {List} receivers 接收人
        @apiParam {Object} convergence_config 收敛策略
        @apiParam {Number} convergence_config.duration 触发时间周期范围
        @apiParam {Number} convergence_config.alert_threshold 周期内触发次数
        @apiParam {Number} convergence_config.mask_time 屏蔽时间

        @apiParamExample {json} 参数样例:
        {
            "rule_name": "规则一",
            "rule_description": "规则一的描述",
            "rule_template_id": 1
            "rule_config": [
                {
                    "metric": {
                        "metric_type": "data_flow",
                        "metric_name": "output_count"
                    },
                    "function": "greater",
                    "constant": {
                        "constant_type": "float",
                        "constant_value": 100
                    },
                    "operation": "and"
                },
                {
                    "metric": {
                        "metric_type": "data_profiling",
                        "metric_name": "null_rate",
                        "metric_field": "field1"
                    },
                    "function": "less",
                    "constant": {
                        "constant_type": "float",
                        "constant_value": 100
                    },
                    "operation": null
                }
            ],
            "rule_config_alias": "(输出量 > 100) & (field1 空值率 < 100)",
            "event_name": "rule1_event",
            "event_alias": "规则一的事件",
            "event_description": "规则一事件的描述",
            "event_type": "data_quality",
            "event_sub_type": "",
            "event_polarity": "positive",
            "event_currency": 30,
            "sensitivity": "private",
            "event_detail_template": "",
            "notify_ways": ["wechart"],
            "receivers": ["admin"],
            "convergence_config": {
                "duration": 1,
                "alert_threshold": 1,
                "mask_time": 60
            }
        }

        @apiSuccessExample Success-Response:
            HTTP/1.1 200 OK
            {
                "errors": {},
                "message": "ok",
                "code": "1500200",
                "result": true,
                "data": true
            }
        """
        bk_username = get_request_username()
        try:
            rule = DataQualityAuditRule.objects.get(id=rule_id)
        except DataQualityAuditRule.DoesNotExist:
            raise dm_pro_errors.AuditRuleNotExistError()

        task_status = "waiting"
        try:
            tasks = DataQualityAuditTask.objects.filter(rule=rule)
            if tasks.count() > 0:
                task_status = tasks[0].status
        except Exception as e:
            logger.error(e)
        if task_status == "running":
            raise dm_pro_errors.RunningAuditTaskCannotEditError()

        try:
            rule_event = DataQualityAuditRuleEvent.objects.filter(rule=rule)[0]
            event = DataQualityEvent.objects.filter(event_id=rule_event.event_id)[0]
        except Exception:
            raise dm_pro_errors.DataQualityEventNotExistError()

        with transaction.atomic(using="bkdata_basic"):
            rule.rule_name = params["rule_name"]
            rule.rule_template_id = params["rule_template_id"]
            rule.rule_config = json.dumps(params["rule_config"])
            rule.rule_config_alias = params["rule_config_alias"]
            rule.updated_by = bk_username
            rule.description = params["rule_description"]
            rule.save()

            event.event_alias = params["event_alias"]
            event.event_currency = params["event_currency"]
            event.event_type = params["event_type"]
            event.event_sub_type = params.get("event_sub_type")
            event.event_polarity = params["event_polarity"]
            event.event_detail_template = params["event_detail_template"]
            event.sensitivity = params["sensitivity"]
            event.updated_by = bk_username
            event.save()

            rule_event.notify_ways = ",".join(params["notify_ways"])
            rule_event.receivers = ",".join(params["receivers"])
            rule_event.convergence_config = json.dumps(params["convergence_config"])
            rule_event.updated_by = bk_username
            rule_event.save()
        return Response(True)

    def delete(self, request, rule_id):
        """
        @api {delete} /datamanage/dataquality/rules/{rule_id}/ 删除数据质量审核规则

        @apiVersion 3.5.0
        @apiGroup DataQualityRule
        @apiName dataquality_audit_rules_delete
        @apiDescription 删除数据质量审核规则

        @apiSuccessExample Success-Response:
            HTTP/1.1 200 OK
            {
                "errors": {},
                "message": "ok",
                "code": "1500200",
                "result": true,
                "data": true
            }
        """
        try:
            rule = DataQualityAuditRule.objects.get(id=rule_id)
        except DataQualityAuditRule.DoesNotExist:
            raise dm_pro_errors.AuditRuleNotExistError()

        task_status = "waiting"
        try:
            tasks = DataQualityAuditTask.objects.filter(rule=rule)
            if tasks.count() > 0:
                task_status = tasks[0].status
        except Exception as e:
            logger.error(e)
        if task_status == "running":
            raise dm_pro_errors.RunningAuditTaskCannotEditError()

        try:
            rule_event = DataQualityAuditRuleEvent.objects.filter(rule=rule)[0]
            event = DataQualityEvent.objects.filter(event_id=rule_event.event_id)[0]
        except Exception:
            raise dm_pro_errors.DataQualityEventNotExistError()

        with transaction.atomic(using="bkdata_basic"):
            rule_event.delete()
            event.delete()
            rule.delete()
        return Response(True)

    @params_valid(serializer=DataQualityDataSetSerializer)
    def list(self, request, params):
        """
        @api {get} /datamanage/dataquality/rules/ 数据质量审核规则列表

        @apiVersion 3.5.0
        @apiGroup DataQualityRule
        @apiName dataquality_audit_rules_list
        @apiDescription 数据质量审核规则列表

        @apiParam {String} data_set_id 数据集ID

        @apiSuccess (200) {Number} data.rule_id 规则ID
        @apiSuccess (200) {String} data.rule_name 规则名称
        @apiSuccess (200) {String} data.rule_description 规则描述
        @apiSuccess (200) {String} data.rule_template_id 规则模板ID
        @apiSuccess (200) {String} data.rule_template_alias 规则模板别名
        @apiSuccess (200) {String} data.rule_config 规则配置
        @apiSuccess (200) {String} data.rule_config_alias 规则配置别名
        @apiSuccess (200) {String} data.audit_task_status 审核任务状态
        @apiSuccess (200) {String} data.event_id 事件ID
        @apiSuccess (200) {String} data.event_name 事件名称
        @apiSuccess (200) {String} data.event_alias 事件别名
        @apiSuccess (200) {Number} data.event_description 事件描述
        @apiSuccess (200) {String} data.event_type 事件类型
        @apiSuccess (200) {String} data.event_sub_type 事件子类型
        @apiSuccess (200) {String} data.event_polarity 事件极性
        @apiSuccess (200) {Number} data.event_currency 事件时效性
        @apiSuccess (200) {String} data.sensitivity 事件敏感度
        @apiSuccess (200) {String} data.event_detail_template 事件详情模板
        @apiSuccess (200) {List} data.notify_ways 事件通知方式
        @apiSuccess (200) {List} data.receivers 事件通知接收者
        @apiSuccess (200) {Object} data.convergence_config 事件通知收敛配置
        @apiSuccess (200) {String} data.created_by 创建人
        @apiSuccess (200) {String} data.created_at 创建时间
        @apiSuccess (200) {String} data.updated_by 更新人
        @apiSuccess (200) {String} data.updated_at 更新时间

        @apiSuccessExample Success-Response:
            HTTP/1.1 200 OK
            {
                "errors": {},
                "message": "ok",
                "code": "1500200",
                "result": true,
                "data": [
                    {
                        "rule_id": 1,
                        "rule_name": "规则一",
                        "rule_description": "规则一的描述",
                        "rule_template_id": 1,
                        "rule_template_alias": "空值约束",
                        "audit_task_status": "running",
                        "rule_config": [
                            {
                                "metric": {
                                    "metric_type": "data_flow",
                                    "metric_name": "output_count"
                                },
                                "function": "greater",
                                "constant": {
                                    "constant_type": "float",
                                    "constant_value": 100
                                },
                                "operation": "and"
                            },
                            {
                                "metric": {
                                    "metric_type": "data_profiling",
                                    "metric_name": "null_rate",
                                    "metric_field": "field1"
                                },
                                "function": "less",
                                "constant": {
                                    "constant_type": "float",
                                    "constant_value": 100
                                },
                                "operation": null
                            }
                        ],
                        "rule_config_alias": "(输出量 > 100) & (field1 空值率 < 100)",
                        "event_id": "591_rule1_event",
                        "event_name": "rule1_event",
                        "event_alias": "规则一的事件",
                        "event_description": "规则一事件的描述",
                        "event_type": "data_quality",
                        "event_sub_type": "",
                        "event_polarity": "positive",
                        "event_currency": 30,
                        "sensitivity": "private",
                        "event_detail_template": "",
                        "notify_ways": ["wechat"],
                        "receivers": ["admin"],
                        "convergence_config": {
                            "duration": 1,
                            "alert_threshold": 1,
                            "mask_time": 60
                        },
                        "created_by": "admin",
                        "created_at": "2020-07-27 10:30:00",
                        "updated_by": "admin",
                        "updated_at": "2020-07-27 10:31:00"
                    }
                ]
            }
        """
        results = []
        rules = (
            DataQualityAuditRule.objects.select_related("rule_template")
            .filter(data_set_id=params["data_set_id"])
            .order_by("id")
        )
        for rule in rules:
            try:
                rule_event = DataQualityAuditRuleEvent.objects.select_related("event").filter(rule=rule)[0]
            except Exception as e:
                logger.error(e)
                raise dm_pro_errors.AuditRuleEventNotFoundError()

            task_status = "waiting"
            try:
                tasks = DataQualityAuditTask.objects.filter(rule=rule)
                if tasks.count() > 0:
                    task_status = tasks[0].status
            except Exception as e:
                logger.error(e)

            results.append(
                {
                    "rule_id": rule.id,
                    "rule_name": rule.rule_name,
                    "rule_description": rule.description,
                    "rule_template_id": rule.rule_template.id,
                    "rule_template_alias": rule.rule_template.template_alias,
                    "audit_task_status": task_status,
                    "rule_config": json.loads(rule.rule_config),
                    "rule_config_alias": rule.rule_config_alias,
                    "event_id": rule_event.event.event_id,
                    "event_name": rule_event.event.event_name,
                    "event_alias": rule_event.event.event_alias,
                    "event_description": rule_event.event.description,
                    "event_type": rule_event.event.event_type,
                    "event_sub_type": rule_event.event.event_sub_type,
                    "event_polarity": rule_event.event.event_polarity,
                    "event_currency": rule_event.event.event_currency,
                    "sensitivity": rule_event.event.sensitivity,
                    "event_detail_template": rule_event.event.event_detail_template,
                    "notify_ways": rule_event.notify_ways.split(","),
                    "receivers": rule_event.receivers.split(","),
                    "convergence_config": json.loads(rule_event.convergence_config),
                    "created_by": rule.created_by,
                    "created_at": rule.created_at,
                    "updated_by": rule.updated_by,
                    "updated_at": rule.updated_at,
                }
            )

        return Response(results)

    @detail_route(methods=["post"], url_path="start")
    def start(self, request, rule_id):
        """
        @api {post} /datamanage/dataquality/rules/{rule_id}/start/ 启动数据质量审核任务

        @apiVersion 3.5.0
        @apiGroup DataQualityRule
        @apiName dataquality_audit_rule_start
        @apiDescription 启动数据质量审核任务

        @apiSuccessExample Success-Response:
            HTTP/1.1 200 OK
            {
                "errors": {},
                "message": "ok",
                "code": "1500200",
                "result": true,
                "data": true
            }
        """
        bk_username = get_request_username()
        try:
            rule = DataQualityAuditRule.objects.get(id=rule_id)
        except DataQualityAuditRule.DoesNotExist:
            raise dm_pro_errors.AuditRuleNotExistError()

        try:
            rule_event = DataQualityAuditRuleEvent.objects.select_related("event").filter(rule=rule)[0]
        except Exception as e:
            logger.error(e)
            raise dm_pro_errors.AuditRuleEventNotFoundError()

        try:
            task = DataQualityAuditTask.objects.filter(rule=rule)[0]
            if task.status == "running":
                raise dm_pro_errors.AuditTaskAlreadyStartedError()
            task.rule_config = json.dumps(DataQualityAuditTask.convert_task_rule_config(rule, rule_event.event))
            task.rule_config_alias = rule.rule_config_alias
            task.status = "running"
            task.updated_by = bk_username
            task.save()
            self.start_task(task)
        except IndexError:
            task = DataQualityAuditTask(
                data_set_id=rule.data_set_id,
                rule=rule,
                rule_config=json.dumps(DataQualityAuditTask.convert_task_rule_config(rule, rule_event.event)),
                rule_config_alias=rule.rule_config_alias,
                status="running",
                created_by=bk_username,
            )
            task.save()
            self.start_task(task)

        return Response(True)

    def start_task(self, task):
        try:
            result = redis_lpush(
                RULE_AUDIT_TASK_QUEUE,
                json.dumps(
                    {
                        "data_set_id": task.data_set_id,
                        "rule_id": task.rule.id,
                        "rule_config": task.rule_config,
                    }
                ),
            )
            if not result:
                raise dm_pro_errors.StartAuditTaskError()
        except Exception:
            raise dm_pro_errors.StartAuditTaskError()

    @detail_route(methods=["post"], url_path="stop")
    def stop(self, request, rule_id):
        """
        @api {post} /datamanage/dataquality/rules/{rule_id}/stop/ 停止数据质量审核任务

        @apiVersion 3.5.0
        @apiGroup DataQualityRule
        @apiName dataquality_audit_rule_stop
        @apiDescription 停止数据质量审核任务

        @apiSuccessExample Success-Response:
            HTTP/1.1 200 OK
            {
                "errors": {},
                "message": "ok",
                "code": "1500200",
                "result": true,
                "data": true
            }
        """
        bk_username = get_request_username()
        try:
            task = DataQualityAuditTask.objects.filter(rule=rule_id)[0]
            if task.status == "waiting":
                raise dm_pro_errors.AuditTaskAlreadyStoppedError()
            task.status = "waiting"
            task.updated_by = bk_username
            task.save()
        except IndexError:
            raise dm_pro_errors.AuditTaskAlreadyStoppedError()
        return Response(True)

    @detail_route(methods=["get"], url_path="metrics")
    @params_valid(serializer=DataQualityRuleMetricsSerializer)
    def metrics(self, request, rule_id, params):
        """
        @api {get} /datamanage/dataquality/rules/{rule_id}/metrics/ 数据质量审核规则相关指标

        @apiVersion 3.5.0
        @apiGroup DataQualityRule
        @apiName dataquality_audit_rule_metrics
        @apiDescription 数据质量审核规则相关指标

        @apiParam {Number} start_time 开始时间
        @apiParam {Number} end_time 结束时间

        @apiSuccess (200) {object} field1_x 指标信息
        @apiSuccess (200) {object} field1_x.metric_type 指标类型
        @apiSuccess (200) {String} field1_x.metric_name 指标名称
        @apiSuccess (200) {String} field1_x.metric_alias 指标别名
        @apiSuccess (200) {String} field1_x.metric_field 字段名称
        @apiSuccess (200) {List} field1_x.today 指标1今日曲线数据
        @apiSuccess (200) {List} field1_x.last_day 指标1昨日曲线数据
        @apiSuccess (200) {List} field1_x.last_week 指标1上周曲线数据

        @apiSuccessExample Success-Response:
            HTTP/1.1 200 OK
            {
                "errors": {},
                "message": "ok",
                "code": "1500200",
                "result": true,
                "data": {
                    "F1_null_rate": {
                        "metric_type": "data_profiling",
                        "metric_name": "null_rate",
                        "metric_alias": "空值率",
                        "metric_field": "F1",
                        "today": [
                            {
                                "time": 1596470400,
                                "null_rate": 10
                            }
                        ],
                        "last_day": [
                            {
                                "time": 1596470400,
                                "null_rate": 10
                            }
                        ],
                        "last_week": [
                            {
                                "time": 1596470400,
                                "null_rate": 10
                            }
                        ]
                    }
                }
            }
        """
        start_time = params.get("start_time", time.time() - 86400)
        end_time = params.get("end_time", time.time())
        try:
            rule = DataQualityAuditRule.objects.get(id=rule_id)
        except DataQualityAuditRule.DoesNotExist:
            raise dm_pro_errors.AuditRuleNotExistError()

        metrics = DataQualityMetric.objects.filter(active=True)
        metrics_configs = {metric.metric_name: metric for metric in metrics}

        response = {}
        rule_configs = json.loads(rule.rule_config) or []
        for rule_config in rule_configs:
            metric_info = rule_config.get("metric")
            metric_type = metric_info.get("metric_type")
            metric_name = metric_info.get("metric_name")

            try:
                metric_config = metrics_configs[metric_name]
            except Exception as e:
                logger.error(e)
                continue

            if metric_type == "data_profiling":
                metric_field = metric_info.get("metric_field")
                metric_key = "{}_{}".format(metric_field, metric_name)
                response[metric_key] = {
                    "metric_type": metric_type,
                    "metric_name": metric_name,
                    "metric_alias": metric_config.metric_alias,
                    "today": self.get_today_data_profiling_series(
                        metric_config,
                        rule.data_set_id,
                        metric_field,
                        start_time,
                        end_time,
                    ),
                    "last_day": self.get_last_day_data_profiling_series(
                        metric_config,
                        rule.data_set_id,
                        metric_field,
                        start_time,
                        end_time,
                    ),
                    "last_week": self.get_last_week_data_profiling_series(
                        metric_config,
                        rule.data_set_id,
                        metric_field,
                        start_time,
                        end_time,
                    ),
                }
            else:
                response[metric_name] = {
                    "metric_type": metric_type,
                    "metric_name": metric_name,
                    "metric_alias": metric_config.metric_alias,
                    "today": self.get_today_data_flow_series(metric_name, rule.data_set_id, start_time, end_time),
                    "last_day": self.get_last_day_data_flow_series(metric_name, rule.data_set_id, start_time, end_time),
                    "last_week": self.get_last_week_data_flow_series(
                        metric_name, rule.data_set_id, start_time, end_time
                    ),
                }
        return Response(response)

    def get_today_data_flow_series(self, metric_name, data_set_id, start_time, end_time):
        results = self.get_metric_value(
            metric_name,
            data_set_id=data_set_id,
            storage=None,
            start_time="{}s".format(start_time),
            end_time="{}s".format(end_time),
            format="series",
            tags=["storage_cluster_type"],
            time_grain="1m",
        )

        for storage in ["kafka", "hdfs"]:
            for item in results:
                if item["storage_cluster_type"] == storage:
                    return item["series"]
        return []

    def get_last_day_data_flow_series(self, metric_name, data_set_id, start_time, end_time):
        results = self.get_metric_value(
            metric_name,
            data_set_id=data_set_id,
            storage=None,
            start_time="{}s".format(start_time - 86400),
            end_time="{}s".format(end_time - 86400),
            format="series",
            tags=["storage_cluster_type"],
            time_grain="1m",
        )

        for storage in ["kafka", "hdfs"]:
            for item in results:
                if item["storage_cluster_type"] == storage:
                    return item["series"]
        return []

    def get_last_week_data_flow_series(self, metric_name, data_set_id, start_time, end_time):
        results = self.get_metric_value(
            metric_name,
            data_set_id=data_set_id,
            storage=None,
            start_time="{}s".format(start_time - 86400 * 7),
            end_time="{}s".format(end_time - 86400 * 7),
            format="series",
            tags=["storage_cluster_type"],
            time_grain="1m",
        )

        for storage in ["kafka", "hdfs"]:
            for item in results:
                if item["storage_cluster_type"] == storage:
                    return item["series"]
        return []

    def get_today_data_profiling_series(self, metric_info, data_set_id, field, start_time, end_time):
        results = self.get_profiling_metric_value(
            json.loads(metric_info.metric_config),
            data_set_id=data_set_id,
            field=field,
            start_time="{}s".format(start_time),
            end_time="{}s".format(end_time),
            format="series",
            time_grain="1m",
        )
        return results[0]["series"]

    def get_last_day_data_profiling_series(self, metric_info, data_set_id, field, start_time, end_time):
        results = self.get_profiling_metric_value(
            json.loads(metric_info.metric_config),
            data_set_id=data_set_id,
            field=field,
            start_time="{}s".format(start_time - 86400),
            end_time="{}s".format(end_time - 86400),
            format="series",
            time_grain="1m",
        )
        return results[0]["series"]

    def get_last_week_data_profiling_series(self, metric_info, data_set_id, field, start_time, end_time):
        results = self.get_profiling_metric_value(
            json.loads(metric_info.metric_config),
            data_set_id=data_set_id,
            field=field,
            start_time="{}s".format(start_time - 86400 * 7),
            end_time="{}s".format(end_time - 86400 * 7),
            format="series",
            time_grain="1m",
        )
        return results[0]["series"]

    @list_route(methods=["get"], url_path="summary")
    @params_valid(serializer=DataQualityDataSetSerializer)
    def summary(self, request, params):
        """
        @api {get} /datamanage/dataquality/rules/summary/ 数据审核规则信息汇总

        @apiVersion 1.0.0
        @apiGroup DataQualityRule
        @apiName dataquality_rules_summary
        @apiDescription 数据审核规则信息汇总

        @apiParam {String} data_set_id 数据集ID

        @apiSuccess (200) {Number} data.rule_count 审核任务总数
        @apiSuccess (200) {Number} data.running_rule_count 运行中审核任务数量
        @apiSuccess (200) {Number} data.waiting_rule_count 待启动审核任务数量
        @apiSuccess (200) {Number} data.failed_rule_count 失败审核任务数量

        @apiSuccessExample Success-Response:
            HTTP/1.1 200 OK
            {
                "errors": {},
                "message": "ok",
                "code": "1500200",
                "result": true,
                "data": {
                    "rule_count": 10,
                    "running_rule_count": 10,
                    "waiting_rule_count": 10,
                    "failed_rule_count": 10
                }
            }
        """
        data_set_id = params["data_set_id"]

        rules = DataQualityAuditRule.objects.filter(data_set_id=params["data_set_id"])
        tasks = DataQualityAuditTask.objects.filter(data_set_id=data_set_id)

        response = {
            "rule_count": rules.count(),
            "running_rule_count": 0,
            "waiting_rule_count": 0,
            "failed_rule_count": 0,
        }

        for task in tasks:
            if task.status == "running":
                response["running_rule_count"] += 1
            elif task.status == "failed":
                response["failed_rule_count"] += 1
            elif task.status == "waiting":
                response["waiting_rule_count"] += 1

        response["waiting_rule_count"] += rules.count() - tasks.count()
        return Response(response)

class DataQualityRuleTemplateViewSet(BaseMixin, APIModelViewSet):
    model = DataQualityAuditRuleTemplate
    lookup_field = model._meta.pk.name
    filter_backends = (DjangoFilterBackend,)
    pagination_class = DataPageNumberPagination
    serializer_class = DataQualityRuleTemplateSerializer
    ordering_fields = ("id", "created_at")
    ordering = ("-id",)

    def get_queryset(self):
        return self.model.objects.filter(active=True)

    def list(self, request):
        """
        @api {get} /datamanage/dataquality/rule_templates/ 数据质量审核规则模板列表

        @apiVersion 3.5.0
        @apiGroup DataQualityRule
        @apiName dataquality_audit_rule_templates_list
        @apiDescription 数据质量审核规则模板列表

        @apiSuccess (200) {Number} data.template_name 模板名称
        @apiSuccess (200) {String} data.template_alias 模板别名
        @apiSuccess (200) {String} data.template_config 模板配置
        @apiSuccess (200) {String} data.description 模板描述
        @apiSuccess (200) {String} data.created_by 创建人
        @apiSuccess (200) {String} data.created_at 创建时间
        @apiSuccess (200) {String} data.updated_by 更新人
        @apiSuccess (200) {String} data.updated_at 更新时间

        @apiSuccessExample Success-Response:
            HTTP/1.1 200 OK
            {
                "errors": {},
                "message": "ok",
                "code": "1500200",
                "result": true,
                "data": [
                    {
                        "template_id": 1,
                        "template_name": "null_constraint",
                        "template_alias": "空值约束",
                        "template_config": {},
                        "description": "xxxxxx",
                        "created_by": "admin",
                        "created_at": "2020-07-27 10:30:00",
                        "updated_by": "admin",
                        "updated_at": "2020-07-27 10:31:00"
                    }
                ]
            }
        """
        return super(DataQualityRuleTemplateViewSet, self).list(request)

class DataQualityAuditTaskViewSet(BaseMixin, APIModelViewSet):
    model = DataQualityAuditTask
    lookup_field = model._meta.pk.name
    filter_backends = (DjangoFilterBackend,)
    pagination_class = DataPageNumberPagination
    serializer_class = DataQualityAuditTaskSerializer
    search_fields = ("status",)
    ordering_fields = ("id", "created_at")
    ordering = ("-id",)

    def get_queryset(self):
        return self.model.objects.all()

class DataQualityAuditFunctionViewSet(BaseMixin, APIModelViewSet):
    model = DataQualityAuditFunction
    lookup_field = model._meta.pk.name
    filter_backends = (DjangoFilterBackend,)
    pagination_class = DataPageNumberPagination
    serializer_class = DataQualityAuditFunctionSerializer
    ordering_fields = ("id", "created_at")
    ordering = ("-id",)

    def get_queryset(self):
        return self.model.objects.filter(active=True)

    def list(self, request):
        """
        @api {get} /datamanage/dataquality/functions/ 数据质量审核函数列表

        @apiVersion 3.5.0
        @apiGroup DataQualityRule
        @apiName dataquality_audit_functions_list
        @apiDescription 数据质量审核函数列表

        @apiSuccess (200) {Number} data.function_name 函数名称
        @apiSuccess (200) {String} data.function_alias 函数别名
        @apiSuccess (200) {String} data.function_type 函数类型
        @apiSuccess (200) {String} data.description 函数描述
        @apiSuccess (200) {String} data.created_by 创建人
        @apiSuccess (200) {String} data.created_at 创建时间
        @apiSuccess (200) {String} data.updated_by 更新人
        @apiSuccess (200) {String} data.updated_at 更新时间

        @apiSuccessExample Success-Response:
            HTTP/1.1 200 OK
            {
                "errors": {},
                "message": "ok",
                "code": "1500200",
                "result": true,
                "data": [
                    {
                        "function_name": "addition",
                        "function_alias": "+",
                        "function_type": "builtin",
                        "description": "加法",
                        "created_by": "admin",
                        "created_at": "2020-07-27 10:30:00",
                        "updated_by": "admin",
                        "updated_at": "2020-07-27 10:31:00"
                    },
                    {
                        "function_name": "subtraction",
                        "function_alias": "-",
                        "function_type": "builtin",
                        "description": "减法",
                        "created_by": "admin",
                        "created_at": "2020-07-27 10:30:00",
                        "updated_by": "admin",
                        "updated_at": "2020-07-27 10:31:00"
                    },
                    {
                        "function_name": "multiplication",
                        "function_alias": "*",
                        "function_type": "builtin",
                        "description": "乘法",
                        "created_by": "admin",
                        "created_at": "2020-07-27 10:30:00",
                        "updated_by": "admin",
                        "updated_at": "2020-07-27 10:31:00"
                    },
                    {
                        "function_name": "division",
                        "function_alias": "/",
                        "function_type": "builtin",
                        "description": "除法",
                        "created_by": "admin",
                        "created_at": "2020-07-27 10:30:00",
                        "updated_by": "admin",
                        "updated_at": "2020-07-27 10:31:00"
                    }
                ]
            }
        """
        return super(DataQualityAuditFunctionViewSet, self).list(request)


import matplotlib.pyplot as plt
import numpy as np
from numpy import pi, cos, sin

with open("water_data.txt") as f:
        lines = f.readlines()
        E = []
        A = []
        for line in lines:
            vals = line.split("\t")
            E.append(float(vals[0]))
            A.append(float(vals[1]))

Energy, mean_path = np.array(E), np.array(A)

#task 1
def plot_1():
    plt.title('Mean path as a function of Energy in water')
    plt.xlabel('log Energy (Mev)')
    plt.ylabel('Mean path μ/ρ(Cm^2/g)')
    plt.plot(np.log(Energy), mean_path)
    plt.show()

#task 2
def scatter(incomming_angle_deg, incomming_energy):
    hv = incomming_energy                             #energy in Mev
    mc2 = 0.511                                       #electron volt mass (Mev)
    phi = np.deg2rad(incomming_angle_deg)             #radians of incomming angle
    c = 3e8                                           #m/s

    new_hv = hv/(1+(hv/mc2)*(1-cos(phi)))
    return new_hv

Energy_range = np.linspace(1, 100, 1000)
new_energy_30deg = scatter(30, Energy_range)
new_energy_60deg = scatter(35, Energy_range)
new_energy_80deg = scatter(40, Energy_range)

def plot_2():
    plt.plot(Energy_range, new_energy_30deg)
    plt.plot(Energy_range, new_energy_60deg)
    plt.plot(Energy_range, new_energy_80deg)
    plt.legend(['30 degrees', '35 degrees', '40 degrees'])
    plt.xlabel('Incomming energy (Mev)')
    plt.ylabel('scattered energy (Mev)')
    plt.show()

#task 3
def cross_sec(hv, angle_deg, radius):
    phi = np.deg2rad(angle_deg)
    #cross_p = pi*radius**2 * (v_marked/v)**2 * (v_marked/v + v/v_marked - sin(phi)**2)*sin(phi) #with respect to phi
    hv_m = scatter(angle_deg, hv)
    f1 = (hv_m/hv)**2
    f2 = ((hv_m/hv) + (hv/hv_m) - (sin(phi))**2)
    crosso = (radius**2/2) * f1 * f2                    #with respect to omega
    return crosso

r0 = 2.82*1e-13                                         #electron radius in cm
cross_sec = cross_sec(Energy_range, 30, r0)
plt.plot(Energy_range, cross_sec)
plt.show()


# A part of NonVisual Desktop Access (NVDA)
# Copyright (C) 2008-2021 NV Access Limited, James Teh, Michael Curran, Leonard de Ruijter, Reef Turner,
# Julien Cochuyt
# This file may be used under the terms of the GNU General Public License, version 2 or later.
# For more details see: https://www.gnu.org/licenses/gpl-2.0.html

import typing

import appModuleHandler
import api
import controlTypes
import versionInfo
from NVDAObjects.IAccessible import IAccessible
from baseObject import ScriptableObject
import gui
from scriptHandler import script
import speech
import textInfos
import braille
import config
from logHandler import log

if typing.TYPE_CHECKING:
	import inputCore

nvdaMenuIaIdentity = None

class NvdaDialog(IAccessible):
	"""Fix to ensure NVDA message dialogs get reported when they pop up.
	"""

	def _get_presentationType(self):
		presType = super(NvdaDialog, self).presentationType
		# Sometimes, NVDA message dialogs briefly report the invisible state
		# after they're focused.
		# This causes them to be treated as unavailable and they are thus not reported.
		# If this dialog is in the foreground, treat it as content.
		if presType == self.presType_unavailable and self == api.getForegroundObject():
			return self.presType_content
		return presType

class NvdaDialogEmptyDescription(IAccessible):
	"""Fix to ensure the NVDA settings dialog does not run getDialogText including panel descriptions
		when alt+tabbing back to a focused control on a panel with a description. This would result in the
		description being spoken twice.
	"""

	def _get_description(self):
		"""Override the description property (because we can't override the classmethod getDialogText)
			However this will do to ensure that the dialog is described as we wish.
		"""
		return ""

# Translators: The name of a category of NVDA commands.
SCRCAT_PYTHON_CONSOLE = _("Python Console")

class NvdaPythonConsoleUIOutputClear(ScriptableObject):

	# Allow the bound gestures to be edited through the Input Gestures dialog (see L{gui.prePopup})
	isPrevFocusOnNvdaPopup = True

	@script(
		gesture="kb:control+l",
		# Translators: Description of a command to clear the Python Console output pane
		description=_("Clear the output pane"),
		category=SCRCAT_PYTHON_CONSOLE,
	)
	def script_clearOutput(self, gesture: "inputCore.InputGesture"):
		from pythonConsole import consoleUI
		consoleUI.clear()

class NvdaPythonConsoleUIOutputCtrl(ScriptableObject):

	# Allow the bound gestures to be edited through the Input Gestures dialog (see L{gui.prePopup})
	isPrevFocusOnNvdaPopup = True

	@script(
		gesture="kb:alt+downArrow",
		# Translators: Description of a command to move to the next result in the Python Console output pane
		description=_("Move to the next result"),
		category=SCRCAT_PYTHON_CONSOLE
	)
	def script_moveToNextResult(self, gesture: "inputCore.InputGesture"):
		self._resultNavHelper(direction="next", select=False)

	@script(
		gesture="kb:alt+upArrow",
		# Translators: Description of a command to move to the previous result
		# in the Python Console output pane
		description=_("Move to the previous result"),
		category=SCRCAT_PYTHON_CONSOLE
	)
	def script_moveToPrevResult(self, gesture: "inputCore.InputGesture"):
		self._resultNavHelper(direction="previous", select=False)

	@script(
		gesture="kb:alt+downArrow+shift",
		# Translators: Description of a command to select from the current caret position to the end
		# of the current result in the Python Console output pane
		description=_("Select until the end of the current result"),
		category=SCRCAT_PYTHON_CONSOLE
	)
	def script_selectToResultEnd(self, gesture: "inputCore.InputGesture"):
		self._resultNavHelper(direction="next", select=True)

	@script(
		gesture="kb:alt+shift+upArrow",
		# Translators: Description of a command to select from the current caret position to the start
		# of the current result in the Python Console output pane
		description=_("Select until the start of the current result"),
		category=SCRCAT_PYTHON_CONSOLE
	)
	def script_selectToResultStart(self, gesture: "inputCore.InputGesture"):
		self._resultNavHelper(direction="previous", select=True)

	def _resultNavHelper(self, direction: str = "next", select: bool = False):
		from pythonConsole import consoleUI
		startPos, endPos = consoleUI.outputCtrl.GetSelection()
		if self.isTextSelectionAnchoredAtStart:
			curPos = endPos
		else:
			curPos = startPos
		if direction == "previous":
			for pos in reversed(consoleUI.outputPositions):
				if pos < curPos:
					break
			else:
				# Translators: Reported when attempting to move to the previous result in the Python Console
				# output pane while there is no previous result.
				speech.speakMessage(_("Top"))
				return
		elif direction == "next":
			for pos in consoleUI.outputPositions:
				if pos > curPos:
					break
			else:
				# Translators: Reported when attempting to move to the next result in the Python Console
				# output pane while there is no next result.
				speech.speakMessage(_("Bottom"))
				return
		else:
			raise ValueError(u"Unexpected direction: {!r}".format(direction))
		if select:
			consoleUI.outputCtrl.Freeze()
			anchorPos = startPos if self.isTextSelectionAnchoredAtStart else endPos
			consoleUI.outputCtrl.SetSelection(anchorPos, pos)
			consoleUI.outputCtrl.Thaw()
			self.detectPossibleSelectionChange()
			self.isTextSelectionAnchoredAtStart = anchorPos < pos
		else:
			consoleUI.outputCtrl.SetSelection(pos, pos)
		info = self.makeTextInfo(textInfos.POSITION_CARET)
		copy = info.copy()
		info.expand(textInfos.UNIT_LINE)
		if (
			copy.move(textInfos.UNIT_CHARACTER, 4, endPoint="end") == 4
			and copy.text == ">>> "
		):
			info.move(textInfos.UNIT_CHARACTER, 4, endPoint="start")
		speech.speakTextInfo(info, reason=controlTypes.OutputReason.CARET)

class AppModule(appModuleHandler.AppModule):
	# The configuration profile that has been previously edited.
	# This ought to be a class property.
	oldProfile = None

	@classmethod
	def handlePossibleProfileSwitch(cls):
		from gui.settingsDialogs import NvdaSettingsDialogActiveConfigProfile as newProfile
		if (cls.oldProfile and newProfile and newProfile != cls.oldProfile):
			# Translators: A message announcing what configuration profile is currently being edited.
			speech.speakMessage(_("Editing profile {profile}").format(profile=newProfile))
		cls.oldProfile = newProfile

	def event_appModule_loseFocus(self):
		self.oldProfile = None

	def isNvdaMenu(self, obj):
		global nvdaMenuIaIdentity
		if not isinstance(obj, IAccessible):
			return False
		if nvdaMenuIaIdentity and obj.IAccessibleIdentity == nvdaMenuIaIdentity:
			return True
		if nvdaMenuIaIdentity is not True:
			return False
		# nvdaMenuIaIdentity is True, so the next menu we encounter is the NVDA menu.
		if obj.role == controlTypes.Role.POPUPMENU:
			nvdaMenuIaIdentity = obj.IAccessibleIdentity
			return True
		return False

	def event_NVDAObject_init(self, obj):
		# It seems that context menus always get the name "context" and this cannot be overridden.
		# Fudge the name of the NVDA system tray menu to make it more friendly.
		if self.isNvdaMenu(obj):
			obj.name=versionInfo.name

	def event_gainFocus(self, obj, nextHandler):
		if obj.role == controlTypes.Role.UNKNOWN and controlTypes.State.INVISIBLE in obj.states:
			return
		nextHandler()

	# Silence invisible unknowns for stateChange as well.
	event_stateChange = event_gainFocus

	def event_foreground(self, obj, nextHandler):
		if not gui.shouldConfigProfileTriggersBeSuspended():
			config.conf.resumeProfileTriggers()
		else:
			self.handlePossibleProfileSwitch()
		nextHandler()

	def event_nameChange(self, obj, nextHandler):
		self.handlePossibleProfileSwitch()
		nextHandler()

	def isNvdaSettingsDialog(self, obj):
		if not isinstance(obj, IAccessible):
			return False
		windowHandle = obj.windowHandle
		from gui.settingsDialogs import NvdaSettingsDialogWindowHandle
		if windowHandle == NvdaSettingsDialogWindowHandle:
			return True
		return False

	def isNvdaPythonConsoleUIInputCtrl(self, obj):
		from pythonConsole import consoleUI
		if not consoleUI:
			return
		return obj.windowHandle == consoleUI.inputCtrl.GetHandle()

	def isNvdaPythonConsoleUIOutputCtrl(self, obj):
		from pythonConsole import consoleUI
		if not consoleUI:
			return
		return obj.windowHandle == consoleUI.outputCtrl.GetHandle()

	def chooseNVDAObjectOverlayClasses(self, obj, clsList):
		if obj.windowClassName == "#32770" and obj.role == controlTypes.Role.DIALOG:
			clsList.insert(0, NvdaDialog)
			if self.isNvdaSettingsDialog(obj):
				clsList.insert(0, NvdaDialogEmptyDescription)
				return
		if self.isNvdaPythonConsoleUIInputCtrl(obj):
			clsList.insert(0, NvdaPythonConsoleUIOutputClear)
		elif self.isNvdaPythonConsoleUIOutputCtrl(obj):
			clsList.insert(0, NvdaPythonConsoleUIOutputClear)
			clsList.insert(0, NvdaPythonConsoleUIOutputCtrl)


### 051920 -- trying to learn a simple dimensionality reduction
### of neural activity that keeps diff trials close and diff stimuli separate
### limited to linear methods for now, but should look into nonlinear

import numpy as np

from sklearn.linear_model import LogisticRegression
from metric_learn import MLKR
from analysis.data_utils import get_imcal_data, spike_counts

### start with setting up data
data, corrs = get_imcal_data('tang', corrs=True, elecs=False,
        names=False, sort='batch') # returns as 1-element list

# want consistent neurons
# cause we need as many trials as possible, can use less cells
min_corrs = corrs.min((0, 1))
data = [[image[:, min_corrs > 0.7] for image in day] for day in data]

# early response (for now)
data = [spike_counts(sess, 540, 640) for sess in data]
data = np.concatenate(data, 1)

# set up for the metric_learn API
c, tr, n = data.shape
data = np.concatenate(data, 0) # so (CxTr)xN
labels = np.array([[i] * tr for i in range(c)]).flatten()

### now do the metric learning
model = MLKR(n_components=10, init='auto') # could play with this
model.fit(data, labels)

### how to evaluate?
### look at trial-to-trial variability, I guess
### and compare it to that of the original data
data_trans = model.transform(data).reshape(c, tr, 10) # should realign trials
data = data.reshape(c, tr, n)

data_trial_var = data.var(1).mean()
data_stim_var = data.mean(1).var(0).mean()

trans_trial_var = data_trans.var(1).mean()
trans_stim_var = data_trans.mean(1).var(0).mean()

print(f'original trial-to-trial variability is {data_trial_var}')
print(f'original stimulus-based variability is {data_stim_var}')
print(f'transformed trial-to-trial variability is {trans_trial_var}')
print(f'transformed stimulus-based variability is {trans_stim_var}')

### ratio of stim-based variance improves, which is good
### but really want to know what manifolds they lie across -- should be disentangled
### can get an idea of that by stimulus-decoding performance
### (the whole point of this is to find a better code)

data_trans = data_trans.reshape(c * tr, 10)
data = data.reshape(c * tr, n)

# should look into using SVMs, or other standard approaches
std_model = LogisticRegression(max_iter=1000).fit(data, labels)
trans_model = LogisticRegression(max_iter=1000).fit(data_trans, labels)

std_score = std_model.score(data, labels)
trans_score = trans_model.score(data_trans, labels)

print(f'decoding performance for original data is {std_score}')
print(f'decoding performance for transformed data is {trans_score}')

### results (takes a long time to run!)
### decoding actually gets worse in the transformed space :(
### something seems weird, though, since it's almost perfect in the neural space

### need to mess around with normalization before the models, and
### also thinking harder about the dimensionality of the neurons being
### higher than the number of classes -- is that a problem?

### could solve that problem by looking at a normal session (450 images)
### but then you have a lot less trials
### might just be that this approach is pointless too


import matplotlib.pyplot as plt
import numpy as np
from sklearn import svm

f=open('./CSR1.DAT')
trainscale=300
data=[]
flag=0
for line in f.readlines():
    str = line.strip().split("\t")
    tmp = []
    if str[0]=='1':
        flag=1
    if line=='\n':
        flag=0
    if flag==1:
        for i in range(1, len(str)):
            tmp.append(float(str[i]))
        data.append(tmp)

min=min(data)[0]
max=max(data)[0]
scale=max-min

for num in data:
    num[0]=(num[0]-min)/1#scale

ytrain=np.array(data[0:trainscale])
ytest=np.array(data[trainscale:len(data)])

ytrain=ytrain.ravel()
ytest=ytest.ravel()

y=np.array(data)
y=y.ravel()

norm=1
x = np.arange(1/norm, (len(data)+1)/norm, 1/norm)
xtrain=np.arange(1/norm, (trainscale+1)/norm, 1/norm)
xtrain.shape=(-1,1)
#xpredict = np.arange(trainscale+1, len(data)+1, 1)
xpredict = np.arange(1/norm, (len(data)+1)/norm, 1/norm)
xpredict.shape=(-1,1)

xtest = np.arange((trainscale+1)/norm,( len(data)+1)/norm, 1/norm)
xtest.shape=(-1,1)

train=svm.SVR(kernel='rbf', C=1e3, gamma=20)
train.fit(xtrain,ytrain)
ypredict=train.predict(xpredict)

plt.figure(figsize=(21, 8))
plt.plot(xtest, ytest, label="test", color='g')
plt.plot(xpredict, ypredict, label="predict", color='r')
plt.plot(x, data, label="raw", color='b')
plt.legend()
plt.title("SVR")
plt.show()

"""
x = np.arange(1, len(data)+1, 1)
x.reshape(-1,1)
x.tolist()
print(x)

x=[]
for i in range(1,(len(data)+1)):
    x.append(i)

train=svm.SVR()
train.fit(x,data)
print(train.predict([[1]]))

plt.figure(figsize=(21, 8))
plt.plot(x, data, label="CDF", color='g')
plt.title("CDF")
plt.show()

print(data)
print(x)
"""


import pandas as pd
import os
import geohash

path = "./taxi_test" #文件夹目录

def to_split_txt_line(str_list):
    _vehicle_id = int(str_list[0])
    _report_time = str_list[1]
    _latitude = float(str_list[2])
    _longitude = float(str_list[3][:-1])
    return _vehicle_id, _report_time, _latitude, _longitude

def get_filelist(dir):
    Filelist = []
    for home, dirs, files in os.walk(dir):
        for filename in files:
            Filelist.append(os.path.join(home, filename))
            file_name = os.path.join(home, filename)
            f = open(file_name, 'r')
            lines = f.readlines()
            for line in lines:
                str_list = line.split(',')
                vehicle_id, report_time, latitude, longitude = to_split_txt_line(str_list)
                report_time_ = pd.to_datetime(report_time)
                geohash_str = geohash.encode(latitude, longitude)

            print(filename)
            pass

    return Filelist

if __name__ == "__main__":
    filelist = get_filelist(path)
    for i in filelist:
        print(i)


import datetime, time, os, pyautogui, playsound

from PIL import ImageGrab
from functools import partial
ImageGrab.grab = partial(ImageGrab.grab, all_screens=True)

def min(x, y):
    x *= 60
    return (x + y)

updating_Animation = [
    '[.  ]     ',
    '[.. ]     ',
    '[...]     '
]

started = False
updating_Frame = 0
executed = 0
weekday = datetime.date.today().weekday()
currentTime = min(datetime.datetime.now().hour, datetime.datetime.now().minute)
os.system ("cls")

def Start():
    if (weekday >= 5 or Holiday() == True):
        time.sleep(30)
        Discord()
        os.system ('start ""')
        time.sleep(1)
        pyautogui.write('"C:/Program Files (x86)/Microsoft/Edge/Application/msedge.exe"  https://www.youtube.com/ https://www.netflix.com/browse/ https://geizhals.de/' '&& exit')
        pyautogui.press('enter')
        time.sleep(10)
        pyautogui.hotkey('ctrl', 'tab')
        time.sleep(1)
        pyautogui.click(1600, 700)
        time.sleep(5)
        exit()

def CheckHoliday(x, y, _y, z, _z): #year, month, monthEnd, day, dayEnd
    days = (datetime.date(x, y, z) - datetime.date.today()).days
    _days = (datetime.date(x, _y, _z) - datetime.date.today()).days
    if (days <= 0 and _days >= 0):
        return True
    return False

def Holiday():
    if (CheckHoliday(2021, 3, 4, 31, 11)):
        return True
    elif (CheckHoliday(2021, 5, 6, 23, 6)):
        return True
    elif (CheckHoliday(2021, 7, 9, 29, 11)):
        return True
    elif (CheckHoliday(2021, 5, 5, 13, 16)):
        return True
    return False

def Execution(path, x, *screen):
    try:
        execution = pyautogui.locateCenterOnScreen('Images/' + path, region = (screen))
        pyautogui.click(execution.x - 1920, execution.y)
        time.sleep(x)
    except: playsound.playsound('Sounds/Error.mp3')

def Discord():
    Execution('Discord.png', 1, 0, 265, 1400, 840)
    Execution('Channel.png', 0, 0, 265, 1400, 840)

def Join():
    time.sleep(1)
    pyautogui.click(395, 180)
    time.sleep(2)
    pyautogui.click(1150, 780)
    pyautogui.hotkey('ctrl', 'shift', 'tab')
    pyautogui.hotkey('ctrl', 'w')
    playsound.playsound("Sounds/Message.mp3")

def Complete(j):
    Execution( j , 3, 3720, 700, 2500, 840)
    pyautogui.click(1076, 598)
    pyautogui.click(1220, 700)
    pyautogui.hotkey('ctrl', 'w')

def Open_Link(id):
    os.system ('start ""')
    time.sleep(1)
    pyautogui.write('"C:/Program Files (x86)/Microsoft/Edge/Application/msedge.exe" https://04104644.moodle.belwue.de/moodle/' + id +' && exit')
    pyautogui.press('enter')

def BBB_Join():
    time.sleep(2)
    repeat = 0
    if (pyautogui.locateCenterOnScreen('Images/notakepart.png', region = (2390, 700, 860, 900))):
        while pyautogui.locateCenterOnScreen('Images/takepart.png', region = (2390, 700, 860, 900)) == False and repeat <= 50:
            repeat += 1
            time.sleep(5)
        if (repeat > 50):
            playsound.playsound("Sounds/Error.mp3")
            return
        elif (pyautogui.locateCenterOnScreen('Images/takepart.png', region = (2390, 700, 860, 900))):
            Execution('takepart.png', 1, 2390, 700, 860, 900)
            for _ in range(5):
                time.sleep(2)
                if (pyautogui.locateCenterOnScreen('Images/micro.png', region = (2920, 635, 1300, 920))):
                    Execution('micro.png', 2920, 635, 1300, 920)
                    break
            time.sleep(1)
            Join()
        else:
            pyautogui.hotkey('ctrl', 'w')
            return
    else:
        Execution('takepart.png', 10, 2390, 700, 860, 900)
        Execution('micro.png', 3, 2920, 635, 1300, 920)
        Join()

def Attendence_complete():
    time.sleep(3)
    pyautogui.moveTo(100, 100)
    for i in range(5):
        if(pyautogui.locateCenterOnScreen('Images/Capture.png', region = (3820, 640, 2450, 1100))):
            Complete('Capture.png')
            break
        else:
            if(pyautogui.locateCenterOnScreen('Images/Capture2.png', region = (3820, 640, 2450, 1100))):
                Complete('Capture2.png')
                break
            else:
                pyautogui.scroll(-425)
                if (i == 4):
                    playsound.playsound("Sounds/Error.mp3")
                    pyautogui.hotkey('ctrl', 'w')

def BBB_search():
    time.sleep(3)
    for i in range(10):
        if (pyautogui.locateCenterOnScreen('Images/BBB.png', region = (2465, 475, 1360, 1180))):
            Execution('BBB.png', 1, 2465, 475, 1360, 1180)
            break
        else:
            pyautogui.scroll(-425)
            if (i == 9):
                playsound.playsound("Sounds/Error.mp3")
                pyautogui.hotkey('ctrl', 'w')

def Schedule():
    global started
    global weekday
    if (started == False):
            started = True
            time.sleep(30)
            Discord()
            os.system('start ""')
            time.sleep(1)
            pyautogui.write('"C:/Program Files (x86)/Microsoft/Edge/Application/msedge.exe" https://www.youtube.com/ https://04104644.moodle.belwue.de/moodle/my/ https://www.netflix.com/browse/ https://geizhals.de/ && exit')
            pyautogui.press('enter')
            time.sleep(5)
            pyautogui.hotkey('ctrl', 'tab')
            time.sleep(1)
            pyautogui.click(1040, 630)
            time.sleep(3)
            pyautogui.hotkey('ctrl', 'tab')
            time.sleep(1)
            pyautogui.click(1600, 700)
            pyautogui.hotkey('ctrl', '1')
    elif (weekday == 0):
        Monday()
    elif (weekday == 1):
        Thuesday()
    elif (weekday == 2):
        Wednesday()
    elif (weekday == 3):
        Thursday()
    elif (weekday == 4):
        Friday()

def Monday():
    global executed
    global currentTime
    if (currentTime == min(7,45) and executed != 1):
        executed = 1
        Open_Link('course/view.php?id=208') #Link Reli-Moodleraum
        BBB_search()
        BBB_Join()
        #Open_Link('course/view.php?id=208 && exit') #Link zum Reli-Raum
        #Attendence_complete()
    elif (currentTime == min(9,35) and executed != 2):
        executed = 2
        Open_Link('course/view.php?id=764') #Link zum IMP-Pysikraum
        BBB_search()
        BBB_Join()
        Open_Link('mod/attendance/view.php?id=6599') #Anwesenheitslink IMP Physik
        Attendence_complete()
    elif (currentTime == min(10,20) and executed != 3):
        executed = 3
        Open_Link('course/view.php?id=419') #Link zum IMP-Informatik
        BBB_search()
        BBB_Join()
        Open_Link('mod/attendance/view.php?id=15108') #Anwesenheitslink ImP-Info
        Attendence_complete()
    elif (currentTime == min(11,25) and executed != 4):
        executed = 4
        Open_Link('mod/bigbluebuttonbn/view.php?id=32375') #BBB Englisch
        BBB_Join()
        Open_Link('mod/attendance/view.php?id=12126') #Anwesenheit Englisch
        Attendence_complete()
    elif (currentTime == min(12,10) and executed != 5):
        executed = 5
        Open_Link('mod/bigbluebuttonbn/view.php?id=32576') #BBB Mathe
        BBB_Join()
        Open_Link('mod/attendance/view.php?id=6741') #Anwesenheit Mathe
        Attendence_complete()
    elif (currentTime == min(15,40) and executed != 6):
        executed = 6
        Open_Link('mod/bigbluebuttonbn/view.php?id=35947') #BBB WBS
        BBB_Join()
        exit()

def Thuesday():
    global executed
    global currentTime
    if (currentTime == min(7,45) and executed != 1):
        executed = 1
        Open_Link('mod/bigbluebuttonbn/view.php?id=32907') #BBB Deutsch
        BBB_Join()
    elif (currentTime == min(8,30) and executed != 2):
        executed = 2
        Open_Link('mod/bigbluebuttonbn/view.php?id=23953') #BBB Latein
        BBB_Join()
    elif (currentTime == min(11,25) and executed != 3):
        executed = 3
        Open_Link('course/view.php?id=578') #Link zum Pysikraum
        BBB_search()
        BBB_Join()
        Open_Link('mod/attendance/view.php?id=23299') #Anwesenheit Physik
        Attendence_complete()
    elif (currentTime == min(14,00) and executed != 4):
        executed = 4
        Open_Link('mod/bigbluebuttonbn/view.php?id=31114') #BBB Gk
        BBB_Join()
    elif (currentTime == min(15,40) and executed != 5):
        executed = 5
        Open_Link('mod/bigbluebuttonbn/view.php?id=1221') #BBB Chemie
        BBB_Join()
        Open_Link('mod/attendance/view.php?id=2871') #Anwesenheit Chemie
        Attendence_complete()
        exit()

def Wednesday():
    global executed
    global currentTime
    if (currentTime == min(7,45) and executed != 1):
        executed = 1
        Open_Link('mod/bigbluebuttonbn/view.php?id=1480') #BBB Geschichte
        BBB_Join()
    elif (currentTime == min(9,35) and executed != 2):
        executed = 2
        Open_Link('mod/bigbluebuttonbn/view.php?id=32907') #BBB Deutsch
        BBB_Join()
    elif (currentTime == min(11,25) and executed != 3):
        executed = 3
        Open_Link('mod/bigbluebuttonbn/view.php?id=23953') #BBB Latein
        BBB_Join()
        exit()

def Thursday():
    global executed
    global currentTime
    if (currentTime == min(7,45) and executed != 1):
        executed = 1
        Open_Link('course/view.php?id=901 && exit') #Link zum Bioraum
        BBB_search()
        BBB_Join()
        Open_Link('mod/attendance/view.php?id=16996') #Anwesenheit Bio
        Attendence_complete()
    elif (currentTime == min(9,35) and executed != 2):
        executed = 2
        Open_Link('mod/bigbluebuttonbn/view.php?id=32907') #BBB Deutsch
        BBB_Join()
    elif (currentTime == min(10,20) and executed != 3):
        executed = 3
        Open_Link('mod/bigbluebuttonbn/view.php?id=32576') #BBB Mathe
        BBB_Join()
        Open_Link('mod/attendance/view.php?id=6741') #Anwesenheit Mathe
        Attendence_complete()
    elif (currentTime == min(11,25) and executed != 4):
        executed = 4
        Open_Link('course/view.php?id=312') #Link zum IMP-Matheraum
        BBB_search
        BBB_Join
        #Open_Link('course/view.php?id=312 && exit') #Link zum IMP-Matheraum
        #Attendence_complete()
        exit()

def Friday():
    global executed
    global currentTime
    if (currentTime == min(7,45) and executed != 1):
        executed = 1
        Open_Link('mod/bigbluebuttonbn/view.php?id=32576') #BBB Mathe
        BBB_Join()
        Open_Link('mod/attendance/view.php?id=6741') #Anwesenheit Mathe
        Attendence_complete()
    elif (currentTime == min(9,35) and executed != 2):
        Open_Link('course/view.php?id=888') #Link zum BK-Raum
        executed = 2
        BBB_search()
        BBB_Join()
        Open_Link('mod/attendance/view.php?id=18799') #Anwesenheit BK
        Attendence_complete()
    elif (currentTime == min(11,25) and executed != 3):
        executed = 3
        Open_Link('mod/bigbluebuttonbn/view.php?id=32375') #BBB Englisch
        BBB_Join()
        Open_Link('mod/attendance/view.php?id=12126') #Anwesenheit Englisch
        Attendence_complete()
        exit()

Start()

while True:
    currentTime = min(datetime.datetime.now().hour, datetime.datetime.now().minute)
    Schedule()
    print(updating_Animation[updating_Frame % len(updating_Animation)], end = '\r')
    time.sleep(1)
    updating_Frame += 1


#!/usr/bin/env python

import torch
import torch.nn as nn
from torch.utils.data import DataLoader

from model import VAE
from dataset import ChordDataset

dataset = ChordDataset("../bundle.txt")
loader = DataLoader(dataset, batch_size=4, shuffle=False, collate_fn=lambda x: x)

vae = VAE(len(dataset.alphabet), 16, 16)
vae.load_state_dict(torch.load("vae.model"), strict=False)

for i, batch in enumerate(loader):
  for song in batch:
    if song.size(0):
      song_enc = vae.encode(song)
      print(song.size())
      print(song_enc.size())
      song_dec, _ = vae.decode(song_enc, song.size(0), torch.tensor([dataset.alphabet.index(' ')]))
      print(song_dec.size())
      print(dataset.decode(song_dec))


# https://leetcode.com/problems/remove-linked-list-elements/
# tags: #linked_list
#
# Solution:
# Time complexity: O(n), Space complexity O(1)
from typing import Optional

class ListNode:
    def __init__(self, val=0, next=None):
        self.val = val
        self.next = next

    def __str__(self):
        return str(self.val)

class Solution:
    def removeElements(self, head: Optional[ListNode], val: int) -> Optional[ListNode]:
        dummy = ListNode(-1)
        dummy.next = head
        head = current = dummy

        while current and current.next:
            if current.next.val == val:
                current.next = current.next.next
            else:
                current = current.next
        return head.next

if __name__ == "__main__":
    sol = Solution()

    head_1 = ListNode(1)
    head_1.next = ListNode(2)
    head_1.next.next = ListNode(6)
    head_1.next.next.next = ListNode(3)
    head_1.next.next.next.next = ListNode(4)
    head_1.next.next.next.next.next = ListNode(5)
    head_1.next.next.next.next.next.next = ListNode(6)
    head_1 = sol.removeElements(head=head_1, val=6)  # [1,2,6,3,4,5,6]

    c = head_1
    while c:
        print(c, end=" -> ")
        c = c.next
    print("", end="\n")


x=0
while x <= 5:
    print(x)
    x = x + 1

counties = ["Arapahoe","Denver","Jefferson"]

for county in counties:
    print(county)

for i in range(len(counties)):
    print(counties[i])

counties_dict = {"Arapahoe": 422829, "Denver": 463353, "Jefferson":432438}
for county in counties_dict.keys():
    print(county)

for county, voters in counties_dict.items():
    print(county, voters)

for county in counties_dict:
    print(counties_dict.get(county))

voting_data = [{"county":"Arapahoe", "registered_voters": 422829},
                {"county":"Denver", "registered_voters": 463353},
                {"county":"Jefferson", "registered_voters": 432438}]

for county_dict in voting_data:
    print(county_dict)

for county_dict in voting_data:
    for value in county_dict.values():
        print(value)

#print just the value for the county key
for county_dict in voting_data:
    print(county_dict['county'])

my_votes = int(input("How many votes did you get in the election? "))
total_votes = int(input("What is the total votes in the election? "))
print(f"I received {my_votes / total_votes * 100}% of the total votes.")

#fstring
for county, voters in counties_dict.items():
    print(f"{county} county has {voters} registered voters.")

#fstring for multi-line printout
candidate_votes = int(input("How many votes did the candidate get in the election? "))
total_votes = int(input("What is the total number of votes in the election? "))
message_to_candidate = (
    f"You received {candidate_votes} number of votes. "
    f"The total number of votes in the election was {total_votes}. "
    f"You received {candidate_votes / total_votes * 100}% of the total votes.")

print(message_to_candidate)

#fsting with formatting of floating pt decimal
message_to_candidate = (
    f"You received {candidate_votes:,} number of votes. "
    f"The total number of votes in the election was {total_votes:,}. "
    f"You received {candidate_votes / total_votes * 100:.2f}% of the total votes.")

#importing datetime class
# Import the datetime class from the datetime module.
import datetime
# Use the now() attribute on the datetime class to get the present time.
now = datetime.datetime.now()
# Print the present time.
print("The time right now is ", now)

# Import the datetime class from the datetime module.
import datetime as dt
# Use the now() attribute on the datetime class to get the present time.
now = dt.datetime.now()
# Print the present time.
print("The time right now is ", now)


#!/usr/bin/env python
# -*- coding: utf-8 -*-

import os
from importlib.machinery import SourceFileLoader
try:
    import cleep
except:
    class cleep:
        __file__ = None

def get_core_version_from_sources(repo_dir):
    try:
        cleep_sources = SourceFileLoader("cleep_sources", os.path.join(repo_dir, 'cleep', '__init__.py')).load_module()
        return cleep_sources.__version__
    except:
        return None

MODULES_REPO_URL = {
    'system': 'https://github.com/CleepDevice/cleepapp-system.git',
    'parameters': 'https://github.com/CleepDevice/cleepapp-parameters.git',
    'cleepbus': 'https://github.com/CleepDevice/cleepapp-cleepbus.git',
    'audio': 'https://github.com/CleepDevice/cleepapp-audio.git',
    'network': 'https://github.com/CleepDevice/cleepapp-network.git',
    'update': 'https://github.com/CleepDevice/cleepapp-update.git',
}
DEFAULT_MODULES = list(MODULES_REPO_URL.keys())

GITHUB_ORG = 'CleepDevice'
GITHUB_REPO = 'cleep'
GITHUB_REPO_DOCS = 'cleep-docs'
GITHUB_REPO_APP_DOCS = "cleep-app-docs"

REPO_URL = 'https://github.com/CleepDevice/cleep.git'
REPO_DIR = os.environ.get('REPO_DIR', '/root/cleep-dev')

CORE_SRC = '%s/cleep' % REPO_DIR
CORE_DST = os.path.dirname(cleep.__file__) if cleep.__file__ else None

HTML_SRC = '%s/html' % REPO_DIR
HTML_DST = '/opt/cleep/html'

MODULES_SRC = '%s/modules' % REPO_DIR
MODULES_DST = '%s/modules' % CORE_DST
MODULES_HTML_DST = '%s/js/modules' % HTML_DST
MODULES_SCRIPTS_DST = '/opt/cleep/scripts'

BIN_SRC = '%s/bin' % REPO_DIR
BIN_DST = '/usr/bin'

MEDIA_SRC = '%s/medias' % REPO_DIR
MEDIA_DST = '/opt/cleep'

DOCS_AUTHOR = 'CleepDevice'
DOCS_PROJECT_NAME = 'Cleep core'

CORE_VERSION = get_core_version_from_sources(REPO_DIR)

DEFAULT_RPC_URL = "https://127.0.0.1:443"


##########################################################
# Author : Hwx                                           #
# Date   : 2016-9-30                                     #
# Purpose: KaKs ratio calculation, statistic and figure  #
##########################################################

# import package
import os
import matplotlib
matplotlib.use('Agg')

import numpy             as np
import pandas            as pd
import matplotlib.pyplot as plt

# # def a class
class KaKs_ratio:
    # Initialize the parameters including path and raw data
    def __init__(self):

	# # Work path
        self.path_work = os.getcwd() + "/"

        # # Input path list
        self.path_rawdata_input    = self.path_work + "allSNV.txt"
        print(self.path_rawdata_input)
        print("Start input file")
        # # Read data file
        self.data_rawdata_input    = pd.read_table(self.path_rawdata_input)
        print("Finished raw data input")

        # # Output path list
        self.path_clean_output     = self.path_work + "allSNV_clean.csv"
        self.path_genelist_output  = self.path_work + "allSNV_genelist.csv"
        self.path_snprate_output   = self.path_work + "allSNV_snprate.csv"
        self.path_result_output    = self.path_work + "allSNV_KaKs.csv"
        self.path_figure_output    = self.path_work + "Kaks_genestat.pdf"

    # # Clean the raw data, strip the useless information
    def data_clean(self):

        # # Cleanning raw data
        self.data_clean_1 = self.data_rawdata_input.ix[:,["SNV NO.",
                                                          "Gene",
                                                          "Function",
                                                          "Case mutation",
                                                          "Case(wild|mutation)"
                                                          ]]
        self.data_clean_2 = self.data_clean_1.dropna()
        self.data_clean = self.data_clean_2.sort_index(by='Gene')
        self.data_clean.to_csv(self.path_clean_output)
        print("Finished data clean")

    # # Extract gene list and calculate SNP rate
    def data_preparation(self):

        # # Extract the useful data and acquire the gene list
        try:
            self.data_clean
        except:
            self.data_clean = pd.read_csv(self.path_clean_output)

        self.data_input = self.data_clean.ix[:,["Gene","Function","Case mutation","Case(wild|mutation)"]]
        self.gene_list = self.data_input["Gene"].drop_duplicates()

        # # Extract the mutation information and calculate snp rate of each SNP site
        self.snp_ratio = []
        for wt_snp in self.data_input["Case(wild|mutation)"]:
            wt, snp = wt_snp.split("|")
            self.snp_ratio.append(int(snp)/(int(snp) + int(wt)))
            self.snp_ratio_set = pd.DataFrame(self.snp_ratio,columns=["snp_ratio"])

        self.gene_list.to_csv(self.path_genelist_output)
        self.data_snpratio = pd.concat([self.data_input,self.snp_ratio_set],axis=1)
        self.data_snpratio.to_csv(self.path_snprate_output)
        print("Finished data preparation")

    # # This is core which calcualtes the Ka/Ks ratio
    def calculate_KaKs(self):

        # Prepare Nan data set
        self.ka_ratio = []
        self.ks_ratio = []
        self.ka_count = []
        self.ks_count = []
        self.kaks_ratio_set = []
        self.kaks_count_set = []

        # Extract the gene set and calculate the KaKs rate
        try:
            self.gene_list
        except:
            self.gene_list = pd.read_csv(self.path_genelist_output)

        try:
            self.data_snpratio
        except:
            self.data_snpratio = pd.read_csv(self.path_snprate_output)

        for gene in self.gene_list:
            # Acquire the gene set
            self.gene_set = self.data_snpratio[self.data_snpratio['Gene']==gene]

            # Acquire the total of synonymous and non-synonymous snp rate
            self.sum_snpratio_syn,self.sum_snpratio_nonsyn = self.gene_set[self.gene_set['Function']=='synonymous SNV']['snp_ratio'].sum(),self.gene_set[self.gene_set['Function'] == 'nonsynonymous SNV']['snp_ratio'].sum()
            self.sum_snpcount_syn,self.sum_snpcount_nonsyn = self.gene_set[self.gene_set['Function'] == 'synonymous SNV']['Function'].count(),self.gene_set[self.gene_set['Function'] == 'nonsynonymous SNV']['Function'].count()

            # # This is the very core part: -<| Calculate of KaKs_ratio |>-

            # Calculate Ka/Ks considering the snp distribution in each group
            ka = self.sum_snpratio_nonsyn
            ks = self.sum_snpratio_syn
            self.ka_ratio.append(ka)
            self.ks_ratio.append(ks)
            # print(ka,ks)
            if self.sum_snpratio_syn == 0:
                ks = 0.01
            kaks_ratio = ka / ks
            self.kaks_ratio_set.append(kaks_ratio)

            # Calculate Ka/Ks considering the snp distribution in each group
            ka         = self.sum_snpcount_nonsyn
            ks         = self.sum_snpcount_syn
            if self.sum_snpratio_syn == 0:
                ks = 0.01
            kaks_count = ka / ks
            self.kaks_count_set.append(kaks_count)
        print("Finished calculation")

    # # Summary and output the result
    def result_summary(self):

        # Collect results
        self.result_set = {
            "Gene"       : self.gene_list,
            "Ka"         : self.ka_ratio,
            "Ks"         : self.ks_ratio,
            # "kaks_count" : self.kaks_count_set,
            "kaks_ratio" : self.kaks_ratio_set
        }
        self.result_set = pd.DataFrame(self.result_set)

        # Output the result
        self.result_set = self.result_set.dropna()
        self.result_set.to_csv(self.path_result_output)
        print("Finished summary")

    # # Plot the statistic result of Ka/Ks ratio and genes
    def plot_hist(self):

        # If the result file of KaKs is exist, time can be saved by skipping steps all above and plotting directly
        try:
            self.result_set
        except:
            self.result_set = pd.read_csv(self.path_result_output)
        self.kaks_ratio_set = self.result_set["kaks_ratio"]

        # Acquire x value for x axis
        value_xAxis = []
        for i in self.kaks_ratio_set:
            if i >= 5:
                value_xAxis.append(5)
            else:
                value_xAxis.append(i)
        value_xAxis = pd.Series(value_xAxis).dropna()

        # Set the x axis
        x_max = round((max(self.kaks_ratio_set) * 10 + 1), 0)
        if  x_max >= 5:
            x_max = 52
        bins = []
        for i in range(0, int(x_max), 2):
            bins.append(i * 0.1)

        # Set the hist bins
        hist, bins = np.histogram(value_xAxis, bins=bins)
        width = np.diff(bins)
        center = (bins[:-1] + bins[1:]) / 2

        # Set figure attribute
        fig, ax = plt.subplots(figsize=(20, 10))
        ax.bar(center, hist, align='center', width=width, facecolor='green', edgecolor='white', alpha=0.5)

        # Set figure information
        font_title = {'family': 'serif',
                'color': 'black',
                'weight': 'normal',
                'size': 38,
                }
        font_label = {'family': 'serif',
                'color': 'darkred',
                'weight': 'normal',
                'size': 26,
                }
        ax.set_xticks(bins)
        ax.set_xlabel('Ka/Ks ratio', fontdict=font_label)
        ax.set_ylabel('Amount of specific Ka/Ks', fontdict=font_label)
        ax.set_title('Ka/Ks Pressure selection', fontdict=font_title)

        # Save the figure
        fig.savefig(self.path_figure_output)
        print("Finished plotting histogram")
        plt.show()

if __name__ == "__main__":

    kaks_ratio = KaKs_ratio()

    # kaks_ratio.data_clean()
    # kaks_ratio.data_preparation()
    # kaks_ratio.calculate_KaKs()
    # kaks_ratio.result_summary()
    kaks_ratio.plot_hist()

    print("Finished KaKs Process")


import dataclasses
import functools

from jax_privacy.experiments.image_data import base
from jax_privacy.experiments.image_data import loader
import tensorflow as tf
import tensorflow_datasets as tfds

MEAN_RGB = (0.49139968, 0.48215841, 0.44653091)
STDDEV_RGB = (0.24703223, 0.24348513, 0.26158784)

@dataclasses.dataclass(kw_only=True, slots=True)
class _DatasetConfig(base.ImageDatasetConfig):
  """Builds the input pipeline for MNIST / SVHN / CIFAR.

  Attributes:
    num_samples: Number of examples in the dataset split.
    num_classes: Number of label classes for the dataset.
    split_content: Subset split, e.g. "train[:50000]".
    name: Unique identifying name for the dataset.
    image_size: Image resolution to use.
    using_large_images: Whether to decode images as large images.
    preprocess_name: Name for the preprocessing function.
  """

  num_samples: int
  num_classes: int
  name: str
  split_content: str
  image_size: tuple[int, int]
  preprocess_name: str | None = None

  def _normalize_image(self, image: tf.Tensor) -> tf.Tensor:
    if self.name in ('cifar10', 'cifar100', 'svhn_cropped', 'mnist'):
      if self.name == 'mnist':
        return base.center_image(image)
      elif self.preprocess_name:
        if self.preprocess_name == 'standardise':
          return base.standardize_image_per_channel(
              image,
              mean_per_channel=MEAN_RGB,
              stddev_per_channel=STDDEV_RGB,
          )
        elif self.preprocess_name == 'center':
          return base.center_image(image)
        elif self.preprocess_name == 'none':
          return image
        else:
          raise ValueError(
              'Unexpected preprocessing function: '
              f'{self.preprocess_name}.')
      else:
        return base.standardize_image_per_channel(
            image,
            mean_per_channel=MEAN_RGB,
            stddev_per_channel=STDDEV_RGB,
        )
    else:
      raise ValueError(f'Invalid dataset {self.name}.')

  def _preprocess_label(self, label: tf.Tensor) -> tf.Tensor:
    """Pre-processes the input label."""
    return tf.one_hot(label, depth=self.num_classes)

@dataclasses.dataclass(kw_only=True, slots=True)
class _DataLoader(loader.DataLoader):
  """Data loader for MNIST / CIFAR / SVHN."""

  config: _DatasetConfig

  def load_raw_data(self, shuffle_files: bool) -> tf.data.Dataset:
    ds = tfds.load(
        name=self.config.name,
        split=self.config.split_content,
        shuffle_files=shuffle_files,
    )
    return ds.map(base.DataInputs.from_dict)

MnistLoader = Cifar10Loader = Cifar100Loader = SvhnLoader = _DataLoader

Cifar10TrainConfig = functools.partial(
    _DatasetConfig,
    name='cifar10',
    image_size=(32, 32),
    num_classes=10,
    split_content='train[:45000]',
    num_samples=45_000,
)
Cifar10TrainValidConfig = functools.partial(
    _DatasetConfig,
    name='cifar10',
    image_size=(32, 32),
    num_classes=10,
    split_content='train',
    num_samples=50_000,
)
Cifar10ValidConfig = functools.partial(
    _DatasetConfig,
    name='cifar10',
    image_size=(32, 32),
    num_classes=10,
    split_content='train[45000:]',
    num_samples=5_000,
)
Cifar10TestConfig = functools.partial(
    _DatasetConfig,
    name='cifar10',
    image_size=(32, 32),
    num_classes=10,
    split_content='test',
    num_samples=5_000,
)
Cifar100TrainConfig = functools.partial(
    _DatasetConfig,
    name='cifar100',
    image_size=(32, 32),
    num_classes=100,
    split_content='train[:45000]',
    num_samples=45_000,
)
Cifar100TrainValidConfig = functools.partial(
    _DatasetConfig,
    name='cifar100',
    image_size=(32, 32),
    num_classes=100,
    split_content='train',
    num_samples=50_000,
)
Cifar100ValidConfig = functools.partial(
    _DatasetConfig,
    name='cifar100',
    image_size=(32, 32),
    num_classes=100,
    split_content='train[45000:]',
    num_samples=5_000,
)
Cifar100TestConfig = functools.partial(
    _DatasetConfig,
    name='cifar100',
    image_size=(32, 32),
    num_classes=100,
    split_content='test',
    num_samples=5_000,
)
SvhnTrainConfig = functools.partial(
    _DatasetConfig,
    name='svhn_cropped',
    image_size=(28, 28),
    num_classes=10,
    num_samples=68_257,
    split_content='train[:68257]',
)
SvhnValidConfig = functools.partial(
    _DatasetConfig,
    name='svhn_cropped',
    image_size=(28, 28),
    num_classes=10,
    num_samples=5_000,
    split_content='train[68257:]',
)
SvhnTrainValidConfig = functools.partial(
    _DatasetConfig,
    name='svhn_cropped',
    image_size=(28, 28),
    num_classes=10,
    num_samples=73_257,
    split_content='train',
)
SvhnTestConfig = functools.partial(
    _DatasetConfig,
    name='svhn_cropped',
    image_size=(28, 28),
    num_classes=10,
    num_samples=26_032,
    split_content='test',
)
MnistTrainConfig = functools.partial(
    _DatasetConfig,
    name='mnist',
    image_size=(28, 28),
    num_classes=10,
    num_samples=50_000,
    split_content='train[:50_000]',
)
MnistValidConfig = functools.partial(
    _DatasetConfig,
    name='mnist',
    image_size=(28, 28),
    num_classes=10,
    num_samples=10_000,
    split_content='train[50_000:]',
)
MnistTrainValidConfig = functools.partial(
    _DatasetConfig,
    name='mnist',
    image_size=(28, 28),
    num_classes=10,
    num_samples=60_000,
    split_content='train',
)
MnistTestConfig = functools.partial(
    _DatasetConfig,
    name='mnist',
    image_size=(28, 28),
    num_classes=10,
    num_samples=10_000,
    split_content='test',
)


from splinter import Browser
from bs4 import BeautifulSoup as bs
import pandas as pd
from webdriver_manager.chrome import ChromeDriverManager

def init_browser():
    #NB: Replace path with local path to ChromeDriver
    executable_path = {'executable_path': ChromeDriverManager().install()}
    return Browser('chrome', **executable_path, headless=False)

#Mars Scrape; The Red Planet Shall be Conquered... also watch the Expanse!
def scrape_info():
    browser = init_browser()
    url = "https://mars.nasa.gov/news/"
    browser.visit(url)
    html = browser.html
    soup = bs(html,"html.parser")
    thread = soup.find("div", class_="list_text")
    news_ttl = thread.find("div", class_= "content_title").text
    news_para = thread.find("div", class_="article_teaser_body").text

#image scrape
image_url = "https://www.jpl.nasa.gov/spaceimages/?search=&category=Mars"
browser.visit(image_url)
html = browser.html
soup = bs(html,"html.parser")
thread = soup.find("article", class_="carousel_item")
featured_image = thread.find("a", class_="button fancybox")["data-fancybox-href"]
featured_image_link = "https://www.jpl.nasa.gov/" + featured_image

#hemispheres
hemi_url = "https://astrogeology.usgs.gov/search/results?q=hemisphere+enhanced&k1=target&v1=Mars"
browser.visit(hemi_url)
html = browser.html
soup = bs(html, "html.parser")
results = soup.find_all("div", class_="item")
hemisphere_image_urls = []

 #scrape for mars facts
facts_url = "https://space-facts.com/mars/"
df = pd.read_html(facts_url)
mars_facts = df[0]
mars_facts.columns=["Description", "Mars"]
mars_facts.set_index("Description", inplace=True)
mars_tbl = mars_facts.to_html(classes="table table-bordered")

for result in results:
    title = result.find("h3").text
    hemi_link = result.find("a", class_="itemLink product-item")["href"]
    image_link = "https://astrogeology.usgs.gov/" + hemi_link
    browser.visit(image_link)
    image_html = browser.html
    soup = bs(image_html, "html.parser")
    image_url = "https://astrogeology.usgs.gov/" + soup.find("img", class_="wide-image")["src"]
    hemisphere_image_urls.append({"title":title, "Image_url":image_url})

#create dictionary
mars_dict = {"news_title":news_ttl, "news_text":news_para, "featured_image":featured_image_link , "mars_facts":mars_tbl, "hemi_image_urls":hemisphere_image_urls}

# Close browser session
browser.quit()

#results
return mars_dict


# define seperotr
def seperator():
    print()
    print("<=======================================================================================================================>")

## If you want to scrape a Website ##
#     1. Use The Official API of The Website Owner
#     2. HTML Web Scraping using Some Tools Like bs4

## Installing all the Required Packages ##
# pip install requests -> Install Requests Module
# pip install html5lib -> Install html5lib Module
# pip install bs4 -> Install bs4 Module

## Use of Packages ##
# Request: Scrap Website
# html5lib: Parse Website
# bs4: Used for Using Scraped Website Code

# Importing all the libraies
import requests
from bs4 import BeautifulSoup, NavigableString

url = "https://praddyumnyadav.github.io/Chocolate-Cake-Recipe-HTML./"
url2 = "https://codewithharry.com"
url3 = "https://www.udemy.com"
url4 = "https://www.facebook.com/thechotapacket"

# Welcome for web scraping
print("<======================================================> Web Scraping <======================================================>")
# step 1: Get The HTML
r = requests.get(url2)
r2 = requests.get(url)
htmlContent = r.content
htmlContent2 = r2.content

# step 2: Parse The HTML
soup = BeautifulSoup(htmlContent, "html.parser")
soup2 = BeautifulSoup(htmlContent2, "html.parser")

# step 3: HTML Tree traversal
# Commonly used types of objects in BeautifulSoup:
#     1. Tag
#     2. NavigableString
#     3. BeautifulSoup
#     4. Comment

# Get the title of HTML Page
print("-> Get Title of The HTML Page")
title = soup.title
print(title.string)

# Get all the pragraphs in this page
paras = soup.find_all("p")

# Get all the anchor tags of this page
anchors = soup.find_all("a")

# Get all the links of page
seperator()
print("All Links of The HTML Page .")
allLinks = set()
for link in anchors:
    if link.get("href") != "#":
        allLinks.add(url2 + link.get("href"))

for link in allLinks:
    print(link)

# Get First Element in the HTML Page
seperator()
print("-> Find First Paragraph tag form HTML Page .")
print(soup.find("p"))

# Get Class of any element in the HTML Page
seperator()
print("-> Get Class of any Element in HTML Page")
print(soup.find("p")["class"])

# Find all the elements with class lead
seperator()
print("-> find the paragraph whose class is lead")
print(soup.find_all("p", class_="lead"))

# Get the text from tags/soup
seperator()
print("-> Find Text of Paragraph")
print(soup.find("p").get_text())

# All Text Without Tags of the Page
seperator()
print("-> All Text Without Tags of The Page")
print(soup.get_text())

# All Main Content Of This Page
seperator()
print("-> All Main Content .")
AllMainContent = soup2.find(class_="content")
for element in AllMainContent.children:
    print(element)

# Parent of MainContent
seperator()
print("-> Print Parent of Main Content")
for item in AllMainContent.parents:
    print(item)


import time
import numpy as np
import matplotlib.pyplot as plt

def arrow():
    fig = plt.figure()
    ax = fig.add_subplot(111)

    point = {
        'start': [10, 10],
        'end': [90, 90],
        "v1": [20, 20],
        "v2": [30, 30],
        "v3": [40, 40],
    }

    ax.plot(*point['start'], 'o', color="red")
    ax.plot(*point['end'], 'o', color="blue")

    ax.annotate('', xy=point['end'], xytext=point['start'],
                arrowprops=dict(shrink=0, width=1, headwidth=8,
                                headlength=10, connectionstyle='arc3',
                                facecolor='gray', edgecolor='gray')
                )

    ax.set_xlim([0, 100])
    ax.set_ylim([0, 100])

    plt.show()
    # time.sleep(3)
    # plt.pause(1)
    fig.canvas.draw()
    fig.canvas.flush_events()
    ax.annotate('test', xy=point['v2'], xytext=point['v1'],
                arrowprops=dict(shrink=0, width=1, headwidth=8,
                                headlength=10, connectionstyle='arc3',
                                facecolor='gray', edgecolor='gray')
                )
    fig.canvas.draw()
    fig.canvas.flush_events()
    # plt.pause(1)
    an2: plt.Annotation = ax.annotate('test', xy=point['v3'], xytext=point['v1'],
                                      arrowprops=dict(shrink=0, width=1, headwidth=8,
                                      headlength=10, connectionstyle='arc3',
                                      facecolor='gray', edgecolor='gray')
                                      )
    fig.canvas.draw()
    fig.canvas.flush_events()
    an2.remove()
    # plt.show()
    fig.canvas.draw()
    fig.canvas.flush_events()

if __name__ == "__main__":
    arrow()


# non local imports ^_^
# if your reading this use auto-py-to-exe to convert this to an .exe file
from colorama import Fore, Back, Style
import asyncio
import time
import sys
import socket
import requests

x = 0
responses = []

async def get_isp_from_api(ip):
    url = f"http://ip-api.com/json/{ip}"
    response = requests.get(url)

    if response.status_code == 200:
        data = response.json()
        return data["isp"]
    else:
        print(f"Failed to fetch ISP information. Status code: {response.status_code}")
        return None
    pass

async def ping(host, port, x):
    client_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
    timeout = 1
    try:
        client_socket.settimeout(timeout)

        await asyncio.sleep(1)

        start_time = time.time()
        client_socket.connect((host, port))
        end_time = time.time()

        response_time = (end_time - start_time) * 1000
        responses.insert(x,
                         f"time= {Fore.GREEN}{response_time:.2f}ms{Fore.RESET} - port={Fore.GREEN}{port}{Fore.RESET} ")
        return response_time

    except socket.timeout:
        responses.insert(x,
                         f"{Fore.RED}TIMED OUT{Fore.RESET} - port={Fore.RED}{port}{Fore.RESET} ")
        return None
    except socket.error as e:
        print(f"Error: {e}")
        return None
    finally:
        client_socket.close()

async def startup():

    global x
    host = sys.argv[1]
    ports = [int(c) for c in sys.argv[2].split(',')]
    ip = socket.gethostbyname(host)
    isp = await get_isp_from_api(ip)
    while True:
        for port in ports:
            x = x + 1
            asyncio.create_task(ping(host, port, x))
        all_tasks = asyncio.all_tasks()
        current_task = asyncio.current_task()
        all_tasks.remove(current_task)
        await asyncio.wait(all_tasks)
        print(f"Connected to {Fore.GREEN}{host}{Fore.RESET} ({Fore.MAGENTA}{isp}{Fore.RESET}) {''.join(responses)} ")
        responses.clear()
        x = 0

asyncio.run(startup())