""" Module to take a water_level reading."""

# Raspi-sump, a sump pump monitoring system.
# Al Audet
# http://www.linuxnorth.org/raspi-sump/
#
# All configuration changes should be done in raspisump.conf
# MIT License -- http://www.linuxnorth.org/raspi-sump/license.html
import json
import os
import stat
import time

try:
    import ConfigParser as configparser  # Python2
except ImportError:
    import configparser  # Python3
from hcsr04sensor import sensor
from raspisump import log, alerts, heartbeat

config = configparser.RawConfigParser()
config.read("/home/pi/raspi-sump/raspisump.conf")

configs = {
    "critical_water_level": config.getint("pit", "critical_water_level"),
    "pit_depth": config.getint("pit", "pit_depth"),
    "temperature": config.getint("pit", "temperature"),
    "trig_pin": config.getint("gpio_pins", "trig_pin"),
    "echo_pin": config.getint("gpio_pins", "echo_pin"),
    "unit": config.get("pit", "unit"),
    "initialstate.enabled": config.getint("initialstate", "enabled"),
    "initialstate.bucket_name": config.get("initialstate", "bucket_name"),
    "initialstate.bucket_key": config.get("initialstate", "bucket_key"),
    "initialstate.access_key": config.get("initialstate", "access_key"),
    "initialstate.item_key": config.get("initialstate", "item_key"),
    "jsonfile.enabled": config.getint("jsonfile", "enabled"),
    "jsonfile.path": config.get("jsonfile", "path")
}

# If item in raspisump.conf add to configs dict. If not provide defaults.
try:
    configs["alert_when"] = config.get("pit", "alert_when")
except configparser.NoOptionError:
    configs["alert_when"] = "high"

try:
    configs["heartbeat"] = config.getint("email", "heartbeat")
except configparser.NoOptionError:
    configs["heartbeat"] = 0

def initiate_heartbeat():
    """Initiate the heartbeat email process if needed"""
    if configs["heartbeat"] == 1:
        heartbeat.determine_if_heartbeat()
    else:
        pass

def water_reading():
    """Initiate a water level reading."""
    pit_depth = configs["pit_depth"]
    trig_pin = configs["trig_pin"]
    echo_pin = configs["echo_pin"]
    temperature = configs["temperature"]
    unit = configs["unit"]

    value = sensor.Measurement(trig_pin, echo_pin, temperature, unit)

    try:
        raw_distance = value.raw_distance(sample_wait=0.3)
    except SystemError:
        log.log_errors(
            "**ERROR - Signal not received. Possible cable or sensor problem."
        )
        exit(0)

    return round(value.depth(raw_distance, pit_depth), 1)

def initialstate_stream_reading(reading):
    if "initialstate.enabled" not in configs or configs["initialstate.enabled"] != 1:
        return

    try:
        from ISStreamer.Streamer import Streamer
        streamer = Streamer(bucket_name=configs["initialstate.bucket_name"],
                            bucket_key=configs["initialstate.bucket_key"],
                            access_key=configs["initialstate.access_key"])
        streamer.log(key=configs["initialstate.item_key"], value=reading)
        streamer.flush()
        streamer.close()
        log.log_restarts("sent initialstate reading")
    except Exception as ex:
        log.log_errors("problem streaming reading to initialstate!")

def json_file_reading(reading):
    try:
        if "jsonfile.enabled" not in configs or configs["jsonfile.enabled"] != 1:
            return

        reading_for_json = {"level": reading,
                            "timestamp": time.time() * 1000}

        with open(configs["jsonfile.path"], mode='w') as jsonfile:
            json.dump(reading_for_json, jsonfile)
        os.chmod(configs["jsonfile.path"], stat.S_IRUSR | stat.S_IWUSR | stat.S_IRGRP | stat.S_IROTH)
        log.log_restarts("wrote json file to {filename}".format(filename=configs["jsonfile.path"]))
    except Exception as ex:
        log.log_errors("problem streaming to json file!")

def water_depth():
    """Determine the depth of the water, log result and generate alert
    if needed.
    """

    critical_water_level = configs["critical_water_level"]
    water_depth = water_reading()

    if water_depth < 0.0:
        water_depth = 0.0
    log.log_reading(water_depth)
    initialstate_stream_reading(reading=water_depth)
    json_file_reading(reading=water_depth)

    if water_depth > critical_water_level and configs["alert_when"] == "high":
        alerts.determine_if_alert(water_depth)
    elif water_depth < critical_water_level and configs["alert_when"] == "low":
        alerts.determine_if_alert(water_depth)
    else:
        pass

    initiate_heartbeat()


import datetime
import json
import requests
from matplotlib import pylab

def get_date(url: str):
    """
    返回json格式数据
    :param url: 获取数据API
    :return: dict
    """
    message = ''
    try:
        url = url
        repo_date = json.loads(requests.get(url).text)
        if repo_date['errMsg'] == "9999":
            store_date = {
                'yesterdayBuyNum': repo_date['data']['yesterdayBuyNum'],
                'yesterdayUseNum': repo_date['data']['yesterdayUseNum'],
                'yesterdayUseAmount': repo_date['data']['yesterdayUseAmount'],
                'totalUseAmount': repo_date['data']['totalUseAmount'],
                'totalBuyNum': repo_date['data']['totalBuyNum'],
                'totalUseNum': repo_date['data']['totalUseNum'],
            }
            return store_date
        else:
            message = repo_date['errMsg']
    except:
        raise Exception(message)

def make_text(date: dict):
    """
    返回拼装数据
    :param yesterdayBuyNum: 当日旅游套票销售数量
    :param yesterdayUseNum: 当日使用套票乘车人数
    :param yesterdayUseAmount: 当日使用套票乘车金额
    :param totalUseAmount: 累计使用套票乘车金额
    :param totalBuyNum: 累计旅游套票销售数量
    :param totalUseNum: 累计使用套票乘车人数
    :return: str
    """
    week = {'1': '一', '2': '二', '3': '三', '4': '四', '5': '五', '6': '六', '0': '日'}
    now_date = datetime.datetime.now().strftime('%Y-%m-%d')
    num_week = datetime.datetime.now().strftime('%w')
    now_week = week[num_week]
    title_text = "x旅游套票信息\n当日旅游套票销售数量: {0}套".format(date['yesterdayBuyNum'])
    content_text = ""
    content_text += "当日旅游套票销售数量: {0}套\n" \
                    "当日使用套票乘车人数: {1}人\n" \
                    "当日使用套票乘车金额: {2}元\n" \
                    "------------------------------\n" \
                    "累计旅游套票销售数量: {3}套\n" \
                    "累计使用套票乘车人数: {4}人\n" \
                    "累计使用套票乘车金额: {5}元\n" \
                    "------------------------------\n" \
                    "信息来源: x\n" \
                    "({6} 星期{7})".format(date['yesterdayBuyNum'], date['yesterdayUseNum'], date['yesterdayUseAmount'],
                                         date['totalUseAmount'], date['totalBuyNum'], date['totalUseNum'], now_date,
                                         now_week)
    return (title_text, content_text)

def change_style(title_text: str, content_text: str):
    """
    处理文本转为图片
    :param title_text: 标题
    :param content_text: 文本
    :return: None
    """
    # 图片名
    n_date = datetime.datetime.now().strftime('%Y%m%d%H%M%S')
    pic_name = 'citypass' + n_date + '.png'
    font_title = {
        'family': 'KaiTi',
        'style': 'italic',
        'weight': 'black',
        'color': 'black',
        'size': 24
    }
    font_text = {
        'family': 'KaiTi',
        'style': 'normal',
        'weight': 'normal',
        'color': 'black',
        'size': 20
    }
    pylab.axis('off')
    pylab.text(0, 0.9, title_text, fontdict=font_title)
    pylab.text(0.05, 0, content_text, fontdict=font_text)
    pylab.savefig(pic_name)
    return pic_name

def send_msg(webhook, message: str):
    News = {
        "msgtype": "markdown",
        "markdown": {
            "title": "x数据",
            "text": ">![screenshot](%s)" % (message)
        }
    }
    headers = {'Content-Type': 'application/json'}
    requests.post(webhook, data=json.dumps(News), headers=headers)

if __name__ == "__main__":
    url = "http://x.x.x.x:x/dingdingData"
    res_title_text, res_text = make_text(get_date(url))
    pic_name = change_style(res_title_text, res_text)
    # 配置nginx图片链接
    message = 'http://x.x.x.x:x/citypass/' + pic_name
    # 钉钉机器人测试
    webhook = "https://oapi.dingtalk.com/robot/send?access_token=x"
    send_msg(webhook=webhook, message=message)


# -*- coding: utf-8 -*-
# Copyright (C) 2017, Simone Marsili
# All rights reserved.
# License: BSD 3 clause
# Author: Simone Marsili (simomarsili@gmail.com)
"""
scikit-MSA: sequence data from sequence alignments
=======================================================================

The scikit-MSA (skmsa) package provides a simple interface between
biological sequence data - organized in a multiple sequence alignment
(MSA) file - and the Numpy/Scipy/pandas/scikit-learn ecosystem.
See docstrings for documentation.

Basic usage examples:
>>> import skmsa
>>> msa = skmsa.read('my_alignment_file.fa') # load sequence into a SequenceData object
>>> data = msa.as_array()   # export sequence data as a 2D numpy array
>>> df = msa.as_dataframe() # export sequence data as a pandas dataframe

"""

from __future__ import (absolute_import, division,
                        print_function, unicode_literals)
from builtins import (  # pylint: disable=redefined-builtin, unused-import
    bytes, dict, int, list, object, range, str,
    ascii, chr, hex, input, next, oct, open,
    pow, round, super,
    filter, map, zip)
import time
import logging
from skmsa.sd import SequenceData
from skmsa.parser import Parser
from skmsa.record import (Record, Alignment)
from skmsa.utils import get_version, config_loggers
read = SequenceData.read
load = SequenceData.load

__version__ = get_version()
__copyright__ = 'Copyright (C) 2017 Simone Marsili'
__license__ = 'BSD 3 clause'
__author__ = 'Simone Marsili (simo.marsili@gmail.com)'
__all__ = ['SequenceData',
           'read',
           'load',
           'Parser']

config_loggers()
logger = logging.getLogger(__name__)
now = time.strftime('%Y-%m-%d %H:%M:%S')
header = ('scikit-MSA (%s) - %s' % (__version__, now))
logger.info(header)
logger.info('=' * len(header))
import random
random.seed(1234)


import logging

from flask import Flask, request, jsonify

from hiku.graph import Graph, Root, Field, Option
from hiku.types import Record, Integer, TypeRef, String, Boolean
from hiku.engine import Engine
from hiku.executors.sync import SyncExecutor
from hiku.endpoint.graphql import GraphQLEndpoint

log = logging.getLogger(__name__)

def value_func(fields):
    return ['Hello World!' for _ in fields]

def action_func(fields):
    results = []
    for field in fields:
        print('action performed!', field.options)
        results.append(True)
    return results

DATA_TYPES = {
    'Point': Record[{
        'x': Integer,
        'y': Integer,
    }],
    'Data': Record[{
        'point': TypeRef['Point'],
    }],
}

QUERY_GRAPH = Graph([
    Root([
        Field('value', String, value_func),
    ]),
], data_types=DATA_TYPES)

MUTATION_GRAPH = Graph(QUERY_GRAPH.nodes + [
    Root([
        Field('action', Boolean, action_func,
              options=[Option('data', TypeRef['Data'])]),
    ]),
], data_types=DATA_TYPES)

app = Flask(__name__)

graphql_endpoint = GraphQLEndpoint(
    Engine(SyncExecutor()), QUERY_GRAPH, MUTATION_GRAPH,
)

@app.route('/graphql', methods={'POST'})
def handle_graphql():
    data = request.get_json()
    result = graphql_endpoint.dispatch(data)
    return jsonify(result)

def main():
    logging.basicConfig()
    app.run(port=5000)

if __name__ == '__main__':
    main()


import pathlib
from copy import deepcopy
from io import IOBase
from typing import Union, List

import attr
import paralleldots
import requests
import trio
from aylienapiclient import textapi
from requests import HTTPError

from srec_nlp.common import Query, Score
from srec_nlp.common import (_parallel, _parallel_async, _ensure_utf)
from srec_nlp.exc import RateError

class NLP:
	@_parallel
	@_ensure_utf
	def sentiment(self, text: Union[str, List[str], IOBase, pathlib.Path]) -> Query:
		"""
		:param text: Union[IOBase, str, Path, bytes] a file, pathtofile, bytes or string with the text.
					Function guarantees the value will be converted to text
		:return: Query
		"""
		raise NotImplemented

	@_parallel_async
	async def sentiment_async(self, text: Union[str, List[str], IOBase, pathlib.Path]):
		"""
		:param text: Union[IOBase, str, Path, bytes] a file, pathtofile, bytes or string with the text.
					Function guarantees the value will be converted to text
		:return: Query
		"""
		return await trio.run_sync_in_worker_thread(self.sentiment, text)

	def copy(self):
		return deepcopy(self)

@attr.s
class ParallelDots(NLP):
	"""
	   with ParallelDots(api_key) as nlp:
			   nlp.sentiment(text)
			   nlp.sentiment(bytes)
			   nlp.sentiment(open(path/to/text.txt,'r'))
			   nlp.sentiment(pathlib.Path('path/to/text.txt'))

	   """

	api_key: str = attr.ib()

	@api_key.validator
	def setkey(self, _, value):
		paralleldots.set_api_key(value)

	def __enter__(self):
		return self

	@_parallel
	@_ensure_utf
	def sentiment(self, text: Union[str, List[str], IOBase, pathlib.Path]) -> Query:
		"""
		Produces a sentiment score for query text using ParallelDots API
		:param text: Union[IOBase, str, Path, bytes] a file, pathtofile, bytes or string with the text.
					Function guarantees the value will be converted to text
		:return: Query
		"""
		try:
			response = paralleldots.sentiment(text)
			return Query(text, Score.fromtriplet(**response["sentiment"]))
		except KeyError as kerr:
			assert response
			if response.get('code', 0) == 429:
				raise RateError()
			else:
				print(response)
				raise kerr
		except HTTPError as err:
			print(response)
			raise err

	def __exit__(self, *exc):
		pass

class TextProcessing(NLP):
	"""
	with TextProcessing() as nlp:
			nlp.sentiment(text)
			nlp.sentiment(bytes)
			nlp.sentiment(open(path/to/text.txt,'r'))
			nlp.sentiment(pathlib.Path('path/to/text.txt'))

		# TextProcessing is still usable at this point.
		# So reusing it is still valid

		nlp.sentiment(text)
		nlp.sentiment(bytes)
		nlp.sentiment(open(path/to/text.txt,'r'))
		nlp.sentiment(pathlib.Path('path/to/text.txt'))
	"""

	_url = "http://text-processing.com/api/sentiment/"

	def __enter__(self):
		return self

	@_parallel
	@_ensure_utf
	def sentiment(self, text: Union[str, List[str], IOBase, pathlib.Path]) -> Query:
		"""
		Produces a sentiment score for query text using TextProcessing API
		:param text: Union[IOBase, str, Path, bytes] a file, pathtofile, bytes or string with the text.
					Function guarantees the value will be converted to text
		:return: Query
		"""
		data = dict(text=text)
		result = requests.post(self._url, data=data).json()
		result = result["probability"]
		result["positive"] = result["pos"]
		result["negative"] = result["neg"]
		del result["pos"]
		del result["neg"]
		try:
			return Query(content=text, score=Score.fromtriplet(**result))
		except (HTTPError, KeyError) as _:
			raise HTTPError()

	def __exit__(self, *exc):
		pass

try:
	try:
		import fastText
	except:
		import fasttext as fastText

	class FastText(NLP):
		"""
		model: str/pathlib.Path  the path to the model.

		# Usage example:

		with FastText(path/to/model.bin) as nlp:
			nlp.sentiment(text)
			nlp.sentiment(bytes).. Trio based queries
			nlp.sentiment(open(path/to/text.txt,'r'))
			nlp.sentiment(pathlib.Path('path/to/text.txt'))

		# at this point, nlp.sentiment WILL RAISE AN ERROR because the model has been
		# dealloc'd, to avoid this, use the following:

		nlp = FastText(path/to/model.bin)
		nlp.sentiment(text)
		nlp.sentiment(bytes)
		nlp.sentiment(open(path/to/text.txt,'r'))
		nlp.sentiment(pathlib.Path('path/to/text.txt'))

		"""

		def __init__(self, path: Union[str, pathlib.Path]):
			path = str(path).strip()
			path = pathlib.Path(path)
			self.path = path
			self.model = FastText._load_model(self.path)

		def __enter__(self):
			return self

		def __exit__(self, *vargs, **kwargs):
			del self.model

		@_parallel
		@_ensure_utf
		def sentiment(self, text: str) -> Query:
			"""
			Produces a sentiment score for query text using FastText API
			Note that FastText needs to be installed for this to work.
			:param text: Union[IOBase, str, Path, bytes] a file, pathtofile, bytes or string with the text.
						Function guarantees the value will be converted to text
			:return: Query
			"""
			labels, scores = self.model.predict(text, 1)
			label = labels[0]
			score = scores[0]
			if label == "__label__1":
				score = 1 - score
			score = Score.fromsingle(score)
			return Query(text, score)

		@staticmethod
		def _load_model(path: Union[str, pathlib.Path]):
			p = pathlib.Path(path).expanduser()
			if not p.is_file():
				raise ValueError(f"{p} is not a file.")
			try:
				return fastText.load_model(str(p))
			except Exception as e:
				print(e)
				raise ValueError(f"{p} is not a valid FastText model")

		def copy(self):
			return FastText(self.path)

except ImportError:
	pass

class Aylien(NLP):
	def __init__(self, appid, appkey):
		"""
		:param appid: string Aylien application id
		:param appkey: string Aylien application key

		with Aylien(id, key) as nlp:
			nlp.sentiment(text)
			nlp.sentiment(bytes)
			nlp.sentiment(open(path/to/text.txt,'r'))
			nlp.sentiment(pathlib.Path('path/to/text.txt'))

		# Aylien is still usable at this point.
		# So reusing it is still valid

		nlp.sentiment(text)
		nlp.sentiment(bytes)
		nlp.sentiment(open(path/to/text.txt,'r'))
		nlp.sentiment(pathlib.Path('path/to/text.txt'))
		"""
		self.client = textapi.Client(applicationId=appid, applicationKey=appkey)

	def __enter__(self):
		return self

	@_parallel
	@_ensure_utf
	def sentiment(self, text: Union[str, List[str], IOBase, pathlib.Path]) -> Query:
		"""
		Produces a sentiment score for query text using Aylien API
		:param text: Union[IOBase, str, Path, bytes] a file, pathtofile, bytes or string with the text.
					Function guarantees the value will be converted to text
		:return: Query
		"""
		sentiment = self.client.Sentiment(dict(text=text))
		try:
			confidence = sentiment["polarity_confidence"]
			klass = sentiment["polarity"]
			remaining = 1 - confidence
			vals = dict(positive=remaining / 2, negative=remaining / 2, neutral=remaining / 2)
			vals[klass] = confidence
			return Query(content=text, score=Score.fromtriplet(**vals))
		except (HTTPError, KeyError) as _:
			raise HTTPError()

	def __exit__(self, *exc):
		pass


from gtts import gTTS
import random
from playsound import playsound
import speech_recognition as sr
import os

class PersonalAssistant:
    def __init__(self, recognizer, microphone, keywordlist):
        self.recognizer = recognizer
        self.microphone = microphone
        self.keywordlist = keywordlist

    def respondHelper(self, keyword):
        prefile = "responsetextfiles/"
        if keyword == "special":
            f = open(prefile + "sassy.txt")
            linenum = random.randint(0, 4)
        elif keyword == "hello":
            f = open(prefile + "greeting.txt")
            linenum = random.randint(0, 11)
        else:
            print("well, the keywords weren't chosen")
            return

        for i, line in enumerate(f):
            if i == linenum:
                return line
            elif i > linenum:
                break
        f.close()

    def listeningNow(self):
        # check that recognizer and microphone arguments are appropriate type
        if not isinstance(self.recognizer, sr.Recognizer):
            raise TypeError("`recognizer` must be `Recognizer` instance")

        if not isinstance(self.microphone, sr.Microphone):
            raise TypeError("`microphone` must be `Microphone` instance")

        # let's have a listen
        with microphone as source:
            self.recognizer.adjust_for_ambient_noise(source)
            audio = self.recognizer.listen(source)
        print("Say something")
        try:
            googleGuess = self.recognizer.recognize_google(audio)
            print("Did you say: " + googleGuess)
        except sr.RequestError:
            # API was unreachable or unresponsive
            print("API unavailable")
            googleGuess = "API unavailable"
        except sr.UnknownValueError:
            # speech was unintelligible
            print("Sorry, I couldn't understand that")
            googleGuess = "Sorry, I couldn't understand that"
        return str(googleGuess)

    def recognize_speech_from_mic(self):
        """Transcribe speech from recorded from `microphone`.

        Returns a dictionary with three keys:
        "success": a boolean indicating whether or not the API request was
                   successful
        "error":   `None` if no error occured, otherwise a string containing
                   an error message if the API could not be reached or
                   speech was unrecognizable
        "transcription": `None` if speech could not be transcribed,
                   otherwise a string containing the transcribed text
        """
        # check that recognizer and microphone arguments are appropriate type
        if not isinstance(recognizer, sr.Recognizer):
            raise TypeError("`recognizer` must be `Recognizer` instance")

        if not isinstance(microphone, sr.Microphone):
            raise TypeError("`microphone` must be `Microphone` instance")

        # adjust the recognizer sensitivity to ambient noise and record audio
        # from the microphone
        with microphone as source:
            recognizer.adjust_for_ambient_noise(source)
            audio = recognizer.listen(source)

        # set up the response object
        response = {
            "success": True,
            "error": None,
            "transcription": None
        }

        # try recognizing the speech in the recording
        # if a RequestError or UnknownValueError exception is caught,
        #     update the response object accordingly
        try:
            response["transcription"] = recognizer.recognize_google(audio)
        except sr.RequestError:
            # API was unreachable or unresponsive
            response["success"] = False
            response["error"] = "API unavailable"
        except sr.UnknownValueError:
            # speech was unintelligible
            response["error"] = "Unable to recognize speech"

        return response

    def respond(self, keyword):
        # here is where we respond
        # we get the phrase from the respondHelper function, passing in the keyword (i.e. "hello")
        phrase = self.respondHelper(keyword)
        tts = gTTS(phrase, 'en')

        # For whatever reason, running the program multiple times
        # with the same response.mp3 file causes it to stop working.
        # It seems changing the file name resolves this issue.
        r1 = str(random.randint(1, 1000000))
        r2 = str(random.randint(1, 1000000))
        randfile = r1 + "response" + r2 + ".mp3"
        tts.save(randfile)
        playsound(randfile)
        # We remember to delete the response file after this program finishes.
        # The deletion probably won't show on the windows explorer window
        # until the program ends (either by error or completion)
        os.remove(randfile)

if __name__ == "__main__":
    # create recognizer and mic instances
    recognizer = sr.Recognizer()
    microphone = sr.Microphone()

    # keywords are special words which we can have special actions for if we hear them
    keywordlist = ["special", "hello"]

    # Lovely name, isn't it?
    cassidy = PersonalAssistant(recognizer, microphone, keywordlist)

    done = False
    while not done:
        for j in range(6):  # we'll loop to keep checking for input
            print("Say something!")
            # here, we call our function to grab our words and transcribe it,
            # passing into "guess" a dictionary of values to work with
            guess = cassidy.recognize_speech_from_mic()
            if guess["transcription"]:
                break
            if not guess["success"]:
                break
            print("I didn't catch that. What did you say?\n")
        # if there was an error, stop break out of loop
        if guess["error"]:
            print("ERROR: {}".format(guess["error"]))
            break

        if guess["transcription"]:  # i.e. if the value of "transcription" != None
            audioTranscribed = str(guess["transcription"])
            print("You said: " + audioTranscribed)
            if audioTranscribed in keywordlist:  # if it's a word in our special keywords list
                for i in range(5):
                    try:
                        cassidy.respond(audioTranscribed)
                        break
                    except:
                        print("response failed, trying again")
                        pass
            elif audioTranscribed == "close program":
                done = True
            # if neither if nor elif, then we'll listen again for more input
            elif "open" in audioTranscribed:
                audioTranscribedList = audioTranscribed.split()
                if audioTranscribedList.__len__() == 2:
                    # I want to search in the command prompt for the applicationName + ".exe" or something
                    pass


from matplotlib.mlab import griddata
import matplotlib.pyplot as pl
import numpy as np

tvec, x, y, vx, vy, rho, press, cs, eps = np.loadtxt('4600_2D.out',unpack=True)
pl.clf()

pl.rcParams['figure.figsize'] = 800./72, 800./72

xi = np.arange(-3,3+1e-14,3.0/999.)
yi = np.arange(-3,3+1e-14,3.0/999.)

zi = griddata(x,y,rho,xi,yi)#,interp='linear')

CS = pl.contour(xi,yi,zi,10,linewidths=0.5,colors='k')
CS = pl.contourf(xi,yi,zi,10,cmap=pl.cm.rainbow,
                  vmax=zi.max(), vmin=zi.min())
pl.colorbar()

pl.scatter(x,y,marker='.',c='b',s=5,zorder=10)
pl.xlim(min(x)*1.05,max(x)*1.05)
pl.ylim(min(y)*1.05,max(y)*1.05)
pl.title('Density Color Plot:  ' r"$t=${:.3f} s".format(tvec[0]))
pl.xlabel(r"$x$")
pl.ylabel(r"$y$")
pl.show()
#pl.savefig('4600_2D_density.pdf')


from django.urls import path
from . import views

app_name = 'django_saml2_auth'

urlpatterns = [
    path(r'acs/', views.acs, name="acs"),
    path(r'welcome/', views.welcome, name="welcome"),
    path(r'denied/', views.denied, name="denied"),
]


from bs4 import BeautifulSoup
from clint.textui import progress
import argparse
import base64
import cfscrape
import requests

parser = argparse.ArgumentParser(description='Given 1 episode URL, download all in the season')
parser.add_argument('--url', help='Specify an episode URL')
parser.add_argument('--path', default='./', help='Specify a download path (defaults to ./)')

args = parser.parse_args()

session = requests.session()
scraper = cfscrape.create_scraper(sess=session)

def getEpisodeSoup(url):
    episodeRequest = scraper.get(url)
    episodeSoup = BeautifulSoup(episodeRequest.content, 'html.parser')

    return episodeSoup

def getEpisodeList(episodeSoup):
    episodeList = []
    episodeOptions = episodeSoup.find(id='selectEpisode').findAll('option')

    for ep in episodeOptions:
        episodeList.append(ep['value'])

    return episodeList

def getURL(episodeSoup):
    return base64.b64decode(episodeSoup.find(id='selectQuality').findAll('option')[0]['value'])

def getFilename(episodeSoup):
    return episodeSoup.find(id='divFileName').contents[2].strip()

def downloadFile(episodeURL, filename):
    episodeDownload = requests.get(episodeURL, stream=True)
    path = '%s/%s.mp4' % (args.path, filename)

    print('Downloading %s ...' % filename)

    with open(path, 'wb') as f:
        total_length = int(episodeDownload.headers.get('content-length'))
        for chunk in progress.bar(episodeDownload.iter_content(chunk_size=1024), expected_size=(total_length/1024) + 1):
            if chunk:
                f.write(chunk)
                f.flush()

    print('Done.')

def main(url):
    rootURL = url.rsplit('/', 1)[0]

    episodeSoup = getEpisodeSoup(url)
    episodeList = getEpisodeList(episodeSoup)

    for episode in episodeList:
        epSoup = getEpisodeSoup('%s/%s' % (rootURL, episode))
        downloadFile(getURL(epSoup), getFilename(epSoup))

if __name__ == "__main__":
    if args.url is None:
        print('No --url provided. See --help for more information.')
        exit()
    else:
        main(args.url)


# from Definitions import *
# from Communication import *

# import sys
# import numpy as np
# import folium
# from PySide2.QtWidgets import (QApplication, QLabel, QPushButton,
#                                QVBoxLayout, QWidget, QtWebEngineWidgets)
# from PySide2.QtCore import Slot, Qt

# class MyWidget(QWidget):
#     def __init__(self):
#         QWidget.__init__(self)

#         self.webView = QtWebEngineWidgets.QWebEngineView(MainWindow)
#         self.webView.setGeometry(10,10,400,200)
#         self.url = QtCore.QUrl.fromLocalFile(r"path-to-map\map.html")
#         self.webView.load(self.url)

#         self.button = QPushButton("Click me!")
#         self.text = QLabel("Hello World")
#         self.text.setAlignment(Qt.AlignCenter)

#         self.layout = QVBoxLayout()
#         self.layout.addWidget(self.webView)
#         self.layout.addWidget(self.button)
#         self.setLayout(self.layout)

# if __name__ == "__main__":
#     map_view = folium.Map(location=[51.5074, 0.1278], control_scale=True, zoom_start = 11) # Uses lat then lon. The bigger the zoom number, the closer in you get
#     map_view.save("map.html")

#     app = QApplication(sys.argv)

#     widget = MyWidget()
#     widget.resize(800, 600)
#     widget.show()

#     sys.exit(app.exec_())

# import wx
# import folium
# import wx.html as html

# class MainFrame(wx.Frame):
#     def __init__(self, parent, title):
#         wx.Frame.__init__(self, parent, -1, title)
#         panel = MainPanel(self)

# class MainPanel(wx.Panel):
#     def __init__(self, frame):
#         wx.Panel.__init__(self, frame)

#         txt_style = wx.VSCROLL|wx.HSCROLL|wx.TE_READONLY|wx.BORDER_SIMPLE
#         self.html = html.HtmlWindow(self, -1, size=(300, 150), style=txt_style)
#         folium_map = folium.Map([48., 5.], tiles='stamentoner', zoom_start=6)
#         folium_map.save("map.html")
#         html.LoadPage("C:\\Users\\Jessica\\andromeda\\Python\\map.html")

# app = wx.App()
# frm = MainFrame(None, "Screen layout")
# frm.Show()
# app.MainLoop()

import sys
import subprocess
import os
import folium
from PyQt5 import QtCore, QtGui, QtWidgets
from PyQt5.QtCore import *
from PyQt5.QtGui import *
from PyQt5.QtWidgets import *
from PyQt5.QtWebEngineWidgets import QWebEngineView as QWebView,QWebEnginePage as QWebPage
from PyQt5.QtWebEngineWidgets import QWebEngineSettings as QWebSettings
from PyQt5.QtWidgets import QApplication, QWidget, QMainWindow

class WebPage(QWebPage):

    def javaScriptConsoleMessage(self, level, msg, linenumber, source_id):
        try:
            # print('%s:%s: %s' % (source_id, linenumber, msg))
            self.message = msg
            print('%s' % msg)
            sys.stdout.flush()
        except OSError:
            pass

    @pyqtSlot(str)
    def print(self, text):
        print('From JS:', text)

    def getMessage(self):
        return self.message

class checkBox():

    def __init__(self, label, region, color='white'):
        self.region = region
        self.label = label

        self.chkbox = QCheckBox(self.label)
        self.chkbox.setStyleSheet("color: white")
        self.chkbox.stateChanged.connect(self.state_changed)

    def state_changed(self):
        print(self.label)
        sys.stdout.flush()

    def getObject(self):
        return self.chkbox

class GUI():

    def __init__(self, page):

        self.window = QWidget()
        self.layout = QGridLayout()
        self.page = page

        self.window.setAutoFillBackground(True)
        p = self.window.palette()
        p.setColor(self.window.backgroundRole(), QColor(56, 68, 90))
        self.window.setPalette(p)

        # app.setStyleSheet('QMainWindow{background-color: darkgray;border: 1px solid black;}')

        # # Create map in GUI
        # self.web = QWebView()
        # self.web.load(QtCore.QUrl.fromLocalFile(os.path.abspath('map.html')))
        # # web.show()

        # Create checkbox
        # self.cb1 = QCheckBox('Set Tree')
        # self.cb1.toggle()
        # self.cb1.stateChanged.connect(self.updateState)

        # make checkbox class with label, regional code, etc.

        chkbox_list = []

        self.layout.addWidget (QPushButton('Set All Trees'), 0, 0, 1, 2)
        self.layout.addWidget (QPushButton('Set Region 1 Trees'), 1, 0, 1, 1)
        self.layout.addWidget (QPushButton('Set Region 2 Trees'), 1, 1, 1, 1)
        for i in range(0,5):
            box = checkBox("Set Tree", 1)
            chkbox_list.append(box)
            self.layout.addWidget (box.getObject(), 2+i, 0, 1, 1)
        for i in range(0,5):
            box = checkBox("Set Tree", 2)
            chkbox_list.append(box)
            self.layout.addWidget (box.getObject(), 2+i, 1, 1, 1)

        # self.layout.addWidget(self.cb1)
        self.createMap()

        label = QLabel()
        pixmap = QPixmap('logo.png')
        label.setPixmap(pixmap.scaled(300, 200, QtCore.Qt.KeepAspectRatio))

        self.layout.addWidget(label, 7, 0, 1, 2)

        self.window.setLayout(self.layout)
        self.window.show()

    def state_changed(self):
        print("Pressed", self.getLabel())
        sys.stdout.flush()

    def createMap(self):
        self.web = QWebView()
        self.web.setPage(self.page)
        self.web.setHtml('''
        <!DOCTYPE html>

            <link rel="stylesheet" href="http://cdn.leafletjs.com/leaflet-0.7.3/leaflet.css" />

        ''')

        # Add QWebView to the layout
        self.layout.addWidget(self.web, 0, 2, 8, 1)

def onNewData(data):
    print("asynchronous", data)

app = QApplication(sys.argv)
app.setStyle('Fusion')
page = WebPage()
gui = GUI(page)
app.exec_()

print(page.getMessage())

        #         <!DOCTYPE html>
        #
        #
        #     <link rel="stylesheet" href="http://cdn.leafletjs.com/leaflet-0.7.3/leaflet.css" />
        #
        #
        #
        #
        #

        #
        #
        #

        # ''')


import time
from collections import defaultdict
from datetime import datetime
from itertools import groupby

import nltk
import numpy as np
from nltk import TweetTokenizer
from nltk.corpus import stopwords
from nltk.tokenize import sent_tokenize

from opframe.db import Session
from opframe.models import search
from opframe.models.search import paginate
from opframe.plugins import plugin
from utils import rescale, render_template

stopwords_en = set(stopwords.words('english'))

# noinspection SpellCheckingInspection
@plugin('DocumentRenderer', 'view_renderer', 'Documents')
def render_documents(form, **kwargs):
    session = Session()
    print(form.values(), kwargs)
    results = search.search(session, **form.values())
    results, page, nb_pages = paginate(results, nb_pages=True, **form.values())
    form.page.data = page
    view = render_template('./templates/search/results_documents.html', form=form, nb_pages=nb_pages, results=results,
                           **kwargs)
    session.close()
    return view

# noinspection SpellCheckingInspection
@plugin('StatisticsRenderer', 'view_renderer', 'Statistics')
def render_statistics(form, **kwargs):
    session = Session()
    results = search.search(session, **form.values())
    # Create the corpus from the results
    tknzr = TweetTokenizer()
    texts = []
    for r in results:
        tokens = []
        for sent in sent_tokenize(r.text.strip()):
            tokens += tknzr.tokenize(sent.strip())
        texts.append(tokens)
    feature_summary = dict()
    feature_map = defaultdict(lambda: [])
    for r in results:
        for f in r.features:
            feature_map[f].append(r.features[f])
    # normalized_feature_map = feature_map
    summary = []
    # nfm = quantile_normalize(feature_map)
    nfm = rescale(feature_map)
    # nfm = feature_map
    for fname, data in nfm.items():
        hist, bin_edges = np.histogram(data, 20)
        hist, bin_edges = hist.tolist(), bin_edges.tolist()
        bins = [dict(x0=x, x1=y, length=z) for x, y, z in zip(bin_edges[:-1], bin_edges[1:], hist)]
        feature_summary[fname] = dict(bins=bins)
        quartiles = np.percentile(data, [25, 50, 75])
        summary += [dict(
            label=fname,
            values=dict(
                Q1=quartiles[0],
                Q2=quartiles[1],
                Q3=quartiles[2],
                min=min(data),
                max=max(data),
                outliers=[]
            )
        )]
    del feature_map
    corpus = nltk.TextCollection(texts)
    corpus.collocations(100)
    collocations = []
    # noinspection PyProtectedMember
    for collocation in corpus._collocations:
        temp = defaultdict(lambda: [])
        for r in results:
            if ' '.join(collocation) in r.text:
                for f in r.features:
                    temp[f].append(r.features[f])
        for k, v in temp.items():
            temp[k] = np.average(v)
        collocations += [(collocation, dict(temp))]
    results = {
        'vocabulary': corpus.vocab().most_common(100),
        'collocations': collocations,
        'features': feature_summary,
        'summary': summary
    }
    view = render_template('./templates/search/results_statistics.html', form=form, results=results, **kwargs)
    session.close()
    return view

# noinspection SpellCheckingInspection
@plugin('WordCloud', 'view_renderer', 'Word Cloud')
def render_wordcloud(form, **kwargs):
    session = Session()
    results = search.search(session, **form.values())
    # Create the corpus from the results
    tknzr = TweetTokenizer()
    texts = []
    for r in results:
        tokens = []
        for sent in sent_tokenize(r.text.strip()):
            tokens += [w for w in tknzr.tokenize(sent.strip()) if w.lower() not in stopwords_en]
        texts.append(tokens)
    corpus = nltk.TextCollection(texts)
    corpus.collocations(100)
    # noinspection PyProtectedMember
    results = {
        'vocabulary': [list(i) for i in corpus.vocab().most_common(1000)],
        'collocations': corpus._collocations,
    }
    view = render_template('./templates/search/results_wordcloud.html', form=form, results=results, **kwargs)
    session.close()
    return view

# noinspection SpellCheckingInspection
@plugin('SpaceTimeRender', 'view_renderer', 'Space Time')
def render_spacetime(form, **kwargs):
    session = Session()
    results = search.search(session, **form.values())
    timeline = []
    for hour, items in groupby(results, lambda x: [x.time.year, x.time.month, x.time.day, x.time.hour, 0, 0]):
        hour_start = time.mktime(datetime(*hour).timetuple())
        hour_end = hour_start + 3600
        timeline.append(
            {'id': hour, 'x': [hour_start, hour_end], 'y': sum(1 for _ in items)}
        )
    location_features = {'type': 'FeatureCollection', 'features': []}
    idx = 0
    for location, items in groupby(results, lambda x: x.location.display_name if x.location else None):
        if location is None or location == '':
            continue
        items = [i for i in items]
        if items[0].location.lat and items[0].location.lon:
            point = [items[0].location.lon, items[0].location.lat]
            location_features['features'].append({
                "type": "Feature",
                "id": '{}'.format(idx),
                'properties': {
                    'name': location,
                    "density": len(items),
                },
                'geometry': {
                    'type': 'Point',
                    'coordinates': point,
                },
            })
        idx += 1
    view = render_template('./templates/search/results_spacetime.html', form=form, **kwargs, timeline=timeline,
                           location_features=location_features)
    session.close()
    return view


from flask import Flask, render_template, request, redirect, g
import nytxw
import sqlite3 as sqlite

DBNAME = 'data/puzzles.db'
app = Flask(__name__)

@app.route("/")
def index():
    return render_template("index.html")

@app.route("/explore")
def explore():
    return render_template("explore.html")

@app.route("/solve")
def solve():
    return render_template("solve.html")

@app.route("/about")
def about():
    return render_template("about.html")

@app.route("/explore/search", methods=["POST"])
def exploresearch():
    search_str = request.form["word"]
    year_str = request.form["year"]
    n_str = request.form["n"]

    # Process search string
    trimmed_search_string = nytxw.process_string_input(search_str)
    # Process year input
    year = nytxw.process_year_input(year_str)
    if year:
        year_as_text = year_str
    else:
        year_as_text = "all years"
    # Process number of results
    if n_str == "all":
        n = 1000
    else:
        n = int(n_str)
    # If there was a search string, search for it!
    if search_str:
        clue_list = nytxw.get_clues_for_word(trimmed_search_string, n, year)
        # If there were results for the search string, show them.
        if clue_list:
            return render_template("explore-result.html", search_type="clues", n=len(clue_list), search_str=trimmed_search_string.upper(), year_as_text=year_as_text, clue_list=clue_list)
        # If not, render result template, but with no results and "No results found!" message.
        else:
            return render_template("explore-result.html", search_type="clues", n=0, search_str=trimmed_search_string.upper(), year_as_text=year_as_text, clue_list=clue_list)
    # If nothing was entered, just render the start page.
    else:
        return render_template("explore.html")

@app.route("/explore/top", methods=["POST"])
def top():
    year_str = request.form["year"]
    n_str = request.form["n"]
    letters_str = request.form["letters"]
    # Process year input
    year = nytxw.process_year_input(year_str)
    if year:
        year_as_text = year_str
    else:
        year_as_text = "all years"
    # Process number of results
    n = int(n_str)
    # Process number of letters
    if letters_str == "all":
        letters = None
        letters_display = "any number of"
    elif letters_str == "15":
        letters = int(letters_str)
        letters_display = "15 or more"
    else:
        letters = int(letters_str)
        letters_display = letters_str

    answer_list = nytxw.get_most_common_answers(length=letters, n=n, year=year)

    return render_template("explore-result.html", search_type="top_answers", n=len(answer_list), letters=letters_display, year_as_text=year_as_text, answer_list=answer_list)

@app.route("/solve/search", methods=["POST"])
def solvesearch():
    search_str = request.form["word"]

    if search_str:
        try:
            result_list = nytxw.search_for_matching_answers(search_str)
            if result_list:
                return render_template("solve-result.html", n=len(result_list), search_str=search_str.upper(), result_list=result_list)
            else:
                return render_template("solve-result.html", n=0, search_str=search_str.upper(), result_list=result_list)
        except:
            return render_template("solve.html")
    else:
        return render_template("solve.html")

if __name__=="__main__":
    app.run(debug=True)


import pandas as pd
import numpy as np
from sklearn.model_selection import StratifiedShuffleSplit

# import csv
_path = 'C:/Users/lxabc/Desktop/QAdata/'
t = pd.read_csv(_path+'ISKG.csv', sep=',', encoding='utf-8', usecols=['_id', '_start', '_end', '_type'], low_memory=False)
t['_id'] = t['_id'].astype(pd.Int32Dtype())
t['_start'] = t['_start'].astype(pd.Int32Dtype())
t['_end'] = t['_end'].astype(pd.Int32Dtype())
print('lines: %d' % t.shape[0])

# split to entites and relations
_idx = t['_start'].first_valid_index()
print('\nrel index start at : %d' % _idx)
entities = t['_id'][:_idx]
triples = t[['_start', '_end', '_type']][_idx:]
print('\nshow entities series: ')
print(entities.head(5))
print('\nshow rels dataframe: ')
print(triples.head(5))

# entities drop duplicates
entities = entities.drop_duplicates()
# print(type(entities)) entities is series
entities = entities.sort_values(ascending=True)
print('\nshow entities series after drop duplicates: ')
print(entities.head(5))

# re-encode entities from 0-n, save as dict
entities.reset_index(drop=True, inplace=True)
ent_dict = entities.to_dict()
ent_dict = dict(zip(ent_dict.values(), ent_dict.keys()))
print('\nshow entities dict after re-encode: ')
print(ent_dict.items())

# re-encode rels
relsLabel = triples['_type'].drop_duplicates()
relsLabel.reset_index(drop=True, inplace=True)
rels_dict = dict(zip(relsLabel.to_dict().values(), relsLabel.to_dict().keys()))
print('\nshow rels dict after re-encode: ')
print(rels_dict.items())

# replace rels token to id
triples['_type'] = triples['_type'].apply(lambda x: rels_dict[x])
print("\nrels id replacement: ")
print(triples.head(5))
print("\nunique rels id: ")
print(triples['_type'].unique())

# replace entities id to new id
triples['_start'] = triples['_start'].apply(lambda x: ent_dict[x])
triples['_end'] = triples['_end'].apply(lambda x: ent_dict[x])
print("\ntriple length: ")
print(triples.shape[0])
print("\ntriples after re-encode: ")
print(triples.head(5))

# save entities/relations/triples encoding
efile = open(_path+'entity2id.txt', 'w')
efile.write(str(len(ent_dict))+'\n')
for k,v in ent_dict.items():
    efile.write(str(k)+'\t'+str(v)+'\n')
efile.close()

rfile = open(_path+'relation2id.txt', 'w', encoding='utf-8-sig')
rfile.write(str(len(rels_dict))+'\n')
for k,v in rels_dict.items():
    rfile.write(str(k)+'\t'+str(v)+'\n')
rfile.close()

tfile = open(_path+'triple2id.txt', 'w')
tfile.write(str(triples.shape[0])+'\n')
tfile.close()
triples.to_csv(_path+'triple2id.txt', sep='\t', header=False,index=False, mode='a')

# split valid and test
t = pd.read_csv(_path+'triple2id.txt', sep='\t', header=None, names=['head', 'tail', 'rel'], encoding='utf-8'
                ,dtype=np.int32, skiprows=1) # skiprows, skip前n行，或者[0, n...]skip第几行

# random_state 可以调整，可以去掉
# iloc是行位置，而loc是dataframe的行index, train_index/test_index返回的是行位置，用iloc
sp = StratifiedShuffleSplit(n_splits = 1,test_size = 0.1,random_state = 0) # 10%
for train_index, test_index in sp.split(t, t['rel']):
    trainds = t.iloc[train_index]
    tvds = t.iloc[test_index]

# rels distribution
print("\ntest/valid rels distribution: ")
print(tvds['rel'].value_counts())

# total triple have entity not in train
# t_ents = trainds['head'].append(trainds['tail']) # 在series中检索会有问题，用unique是准确的
# t_entuni = t_ents.unique()
# t_reluni = trainds['rel'].unique()
trainHeadUni = trainds['head'].unique()
trainTailUni = trainds['tail'].unique()
i = 0
idxlist = []
for idx, row in tvds.iterrows():
    if row['head'] not in trainHeadUni or row['tail'] not in trainTailUni:
        i+=1
        idxlist.append(idx)
print("\ntotal triple have entity not in train: ")
print(i)

# append to train and drop from valid/test
trainds = trainds.append(tvds.loc[idxlist])
tvds = tvds.drop(index=idxlist)

relsdis = tvds['rel'].value_counts()
print("\ntest/valid relation distribution after drop: ")
print(tvds['rel'].value_counts())

# 如果tvds某个关系只有一个三元组，则使用
if 1 in relsdis.values:
    rel = relsdis[relsdis.values==1].index[0] # index返回是个list, 需要取
    add = tvds[tvds['rel']==rel]
    idx = tvds[tvds['rel']==rel].index
    tvds = tvds.drop(index=idx)
    print("\nrels 1 index: ")
    print(idx)

trainds = trainds.sample(frac=1).reset_index(drop=True)
tvds = tvds.sample(frac=1).reset_index(drop=True)

# tvds划分valid和test
spt = StratifiedShuffleSplit(n_splits = 1,test_size = 0.5,random_state = 0) # 50% test
for test_index, valid_index in spt.split(tvds, tvds['rel']):
    testds = tvds.iloc[test_index]
    validds = tvds.iloc[valid_index]

# add un-split line to valid
if 1 in relsdis.values:
    validds = validds.append(add)

# save train/valid/test data set
trfile = open(_path+'train2id.txt', 'w')
trfile.write(str(trainds.shape[0])+'\n')
trfile.close()
trainds.to_csv(_path+'train2id.txt', sep='\t', header=False,index=False, mode='a')

vafile = open(_path+'valid2id.txt', 'w')
vafile.write(str(validds.shape[0])+'\n')
vafile.close()
validds.to_csv(_path+'valid2id.txt', sep='\t', header=False,index=False, mode='a')

tsfile = open(_path+'test2id.txt', 'w')
tsfile.write(str(testds.shape[0])+'\n')
tsfile.close()
testds.to_csv(_path+'test2id.txt', sep='\t', header=False,index=False, mode='a')


# Import dependencies
import os
import csv
import collections
from collections import Counter

#Define PyPoll's variables
voters_cands = []
votes_per_cand = []

#Change directory to the directory of current python script
os.chdir(os.path.dirname(__file__))

#Path to collect data from the Resources folder
election_data_csv = os.path.join("Resources", "election_data.csv")

#Open and read csv
with open(election_data_csv, newline="") as csvfile:

    csv_reader = csv.reader(csvfile, delimiter=",")

    #Read the header row first
    csv_header = next(csvfile)

    #print(f"Header: {csv_header}")
    #This prints -->> Header: Voter ID,Country,Candidate
    #Read through each row of data after the header
    for row in csv_reader:

        voters_cands.append(row[2])

    #Sort the list by default ascending order
    sorted_list = sorted(voters_cands)

    #sorted_list = sorted(voters_candidates, reverse=True)
    #sorted_list.sort(reverse=True)

    #for key, group in groupby(sorted_list):

    #Arrange the sorted list by most common outcomes
    arrange_list = sorted_list

    #count votes per candidate in most common outcome order and append to a list
    count_cand = Counter (arrange_list)
    votes_per_cand.append(count_cand.most_common())

    #calculate the % of votes per candicate in 3 digital points
    for item in votes_per_cand:

        first = format((item[0][1])*100/(sum(count_cand.values())),'.3f')
        second = format((item[1][1])*100/(sum(count_cand.values())),'.3f')
        third = format((item[2][1])*100/(sum(count_cand.values())),'.3f')
        fourth = format((item[3][1])*100/(sum(count_cand.values())),'.3f')

    #print(c.most_common())
    #print(c.values())
    #print(c.keys())
    #print(sum(c.values()))

#Print the analysis to the terminal
print("Election Results")
print("-------------------------")
print(f"Total Votes:  {sum(count_cand.values())}")
print("-------------------------")
print(f"{votes_per_cand[0][0][0]}: {first}% ({votes_per_cand[0][0][1]})")
print(f"{votes_per_cand[0][1][0]}: {second}% ({votes_per_cand[0][1][1]})")
print(f"{votes_per_cand[0][2][0]}: {third}% ({votes_per_cand[0][2][1]})")
print(f"{votes_per_cand[0][3][0]}: {fourth}% ({votes_per_cand[0][3][1]})")
print("-------------------------")
print(f"Winner:  {votes_per_cand[0][0][0]}")
print("-------------------------")

#Export a text file with the results
election_file = os.path.join("analysis", "election_data.txt")
with open(election_file, "w") as output:

    output.write("Election Results\n")
    output.write("-------------------------\n")
    output.write(f"Total Votes:  {sum(count_cand.values())}\n")
    output.write("-------------------------\n")
    output.write(f"{votes_per_cand[0][0][0]}: {first}% ({votes_per_cand[0][0][1]})\n")
    output.write(f"{votes_per_cand[0][1][0]}: {second}% ({votes_per_cand[0][1][1]})\n")
    output.write(f"{votes_per_cand[0][2][0]}: {third}% ({votes_per_cand[0][2][1]})\n")
    output.write(f"{votes_per_cand[0][3][0]}: {fourth}% ({votes_per_cand[0][3][1]})\n")
    output.write("-------------------------\n")
    output.write(f"Winner: {votes_per_cand[0][0][0]}\n")
    output.write("-------------------------\n")


import webapp2 as webapp
from webapp2_extras import jinja2
from google.appengine.ext.db import users
from datetime import date

class BaseHandler(webapp.RequestHandler):

    @webapp.cached_property
    def jinja2(self):
        # Returns a Jinja2 renderer cached in the app registry.
        return jinja2.get_jinja2(app=self.app)

    def render_response(self, _template, **context):
        # Renders a template and writes the result to the response.
        logout = users.create_logout_url('/')
        context['is_admin'] = users.is_current_user_admin()
        context['request'] = self.request
        context['logout'] = logout
        context['current_user'] = users.get_current_user()
        context['today']=date.today().isoformat()
        rv = self.jinja2.render_template(_template, **context)
        self.response.write(rv)


import ray
import time
from math import radians, sin, cos, atan2, sqrt

from caldist.helpers.coord import lat_target, long_target

def haversine(lat1, long1, lat2, long2):
    _R = 6371e3
    delta_long = long2 - long1

    a = (cos(lat2) * sin(delta_long))**2
    b = (cos(lat1) * sin(lat2) - sin(lat1) * cos(lat2) * cos(delta_long))**2
    c = sin(lat1) * sin(lat2)
    d = cos(lat1) * cos(lat2) * cos(delta_long)

    x = c + d
    y = sqrt(a + b)
    z = atan2(y, x)

    return _R * z

@ray.remote
def distribute(awal, akhir):
    partisi = [
        haversine(-6.345960, 106.695097, lat_target[i], long_target[i])
        for i in range(awal, akhir)
    ]

    return partisi[0]

if __name__ == "__main__":
    ray.init()

    iterasi = len(lat_target)  # 10000
    mulai = time.time()

    hasil_id = []
    [
        hasil_id.append(distribute.remote(idx, idx + 1))
        for idx in range(iterasi)
    ]

    durasi = (time.time() - mulai) * 1000 / iterasi

    print(ray.get(hasil_id))
    print(durasi)


# created by victor p - 16/12/2018
from django.urls import path
from .v1 import views as views_v1

urlpatterns_v1_lines = ([
    path('',
         views_v1.LineView.as_view(),
         name='v1_list_create_line'),
    path('<str:pk>',
         views_v1.LineDetailView.as_view(),
         name='v1_detail_line'),

    ], 'lines')

urlpatterns_v1_routes = ([
    path('',
         views_v1.RouteView.as_view(),
         name='v1_list_create_route'),
    path('<str:pk>',
      views_v1.RouteDetailView.as_view(),
      name='v1_detail_route'),
    ], 'routes')


import os
import shutil
import threading
import time
import datetime
import json
import signal
from math import sqrt, log
import random

import numpy as np
import requests
import pandas as pd

import settings2

def keepFileInRange(path, range):
    """
    保留文件夹下多少数据文件
    :return:
    """
    try:
        file_list = os.listdir(path)
        delete_count = len(file_list) - range
        if delete_count <= 0:
            return
        print("需要删除文件个数:%s" % delete_count)
        t1 = time.time()
        for file in file_list[:delete_count]:
            os.remove(os.path.join(path, file))
        print("删除完成耗时:%s" % (time.time() - t1))
    except Exception as e:
        print(e)
        print("路径不存在或者为空")

def machineDataSuperviser(count):
    """
    控制振动数据移动后文件夹下文件数目
    """
    while True:
        for machine in os.listdir(settings2.MACHINE_TOOL_OUTPUT_PATH):
            machine_path = os.path.join(settings2.MACHINE_TOOL_OUTPUT_PATH, machine)
            for tool in os.listdir(machine_path):
                tool_path = os.path.join(machine_path, tool)
                keepFileInRange(tool_path, count)
        time.sleep(count / 10)

def keepRMSData(rms_path, count):
    """
    每个刀号保留count个数目文件
    :param rms_path:
    :param count:
    :return:
    """
    try:
        while True:
            maps = {}
            counts = 0
            for filename in os.listdir(rms_path)[::-1]:
                tool_name = filename.split("-")[0]
                now_count = maps.get(tool_name, 0)
                now_count += 1
                if tool_name in maps.keys():
                    if maps[tool_name] >= count:
                        delete_file = os.path.join(rms_path, filename)
                        os.remove(delete_file)
                        counts += 1
                    else:
                        maps[tool_name] += 1
                else:
                    maps[tool_name] = 1
            print("移除%s个文件" % counts)
            time.sleep(count / 10)
    except Exception as e:
        print(e)

def resultDataSuperviser(count):
    while True:
        for machine in os.listdir(settings2.OUTPUT_BASE_PATH):
            machine_path = os.path.join(settings2.OUTPUT_BASE_PATH, machine)
            for dir_name in os.listdir(machine_path):
                target_path = os.path.join(machine_path, dir_name)
                # print("检查路径->%s"%target_path)
                if dir_name == settings2.VIRBATION_RMS_OUTPUT_PATH_NAME:
                    continue
                keepFileInRange(target_path, count)
        time.sleep(count / 4)

def machineDataSrcSuperviser(count, path):
    while True:
        keepFileInRange(path, count)
        time.sleep(count / 2)

class ProcessFile(threading.Thread):
    def __init__(self, machine_name, machineInfo_input_path, vibrationData_input_path, using_tool_input_path,
                 vibrationData_move_path, result_output_path):
        super().__init__()
        self.machine_name = machine_name

        self.machineInfo_input_path = machineInfo_input_path
        self.vibrationData_input_path = vibrationData_input_path
        self.using_tool_input_path = using_tool_input_path
        self.vibrationData_move_path = vibrationData_move_path
        self.virbation_raw_output_path = os.path.join(result_output_path, settings2.VIRBATION_OUTPUT_PATH_NAME)
        self.load_data_output_path = os.path.join(result_output_path, settings2.LOAD_DATA_OUTPUT_PATH_NAME)
        self.vibration_rms_output_path = os.path.join(result_output_path, settings2.VIRBATION_RMS_OUTPUT_PATH_NAME)
        self.old_tool = 0
        self.usingTool = 0
        self.last_trans_time = self.now
        self.checkPathOrCreate(self.vibrationData_move_path)
        self.checkPathOrCreate(self.virbation_raw_output_path)
        self.checkPathOrCreate(self.load_data_output_path)
        self.checkPathOrCreate(self.vibration_rms_output_path)

        self.hasChangeTool = False
        self.old_vib_data = ""
        self.old_vib_data_count = 0

    def setToolInfo(self):
        """
        获取当前机台加工刀具信息
        :param path:
        :return:
        """

        if not os.path.exists(self.using_tool_input_path):
            print("当前机台加工刀具信息无法确认")
            return False
        path = self.using_tool_input_path

        file_list = os.listdir(path)
        try:
            if file_list:
                tool_info_file = file_list[0]
                temp = tool_info_file.rsplit("-", 1)
                str_time = temp[0]
                tool_name = temp[1].rsplit(".")[0]

                if tool_name == "0" or tool_name == "None":
                    # 说明当前机台正在换刀或者待机中
                    self.usingTool = 0
                    return False
                if not self.usingTool:
                    self.usingTool = tool_name
                elif tool_name != self.usingTool:
                    print("刀具更换:%s->>%s" % (self.usingTool, tool_name))
                    self.old_tool = self.usingTool
                    self.usingTool = tool_name
                    self.hasChangeTool = True

                else:
                    self.hasChangeTool = False
                return True
            else:
                print("路径%s没有文件" % path)
                return False
        except Exception:
            print("机台信息获取失败")

    @staticmethod
    def findLastBeforTime(str_time, files_path=None, p1='%Y-%m-%d-%H-%M-%S-%f', p2='%Y-%m-%d-%H-%M-%S-%f'):
        """
        在files_path中寻找一个文件，这个文件名表示的时间需要比str_time小并且最接近str_time
        算法思路:
            1. 获取路径下所有文件列表 并排序
            2. 把str_time转换为可以比较的形式compile_time
            3. 遍历排序后的列表，其每个元素与compile_time进行一一对比
                1. 如果元素比str_time小继续循环
                2. 如果元素比str_time大：
                    1. 如果元素是第一个元素 则返回-1
                    2. 否则返回上一个元素
            4. 如果遍历完成后还没有返回 则说明所有元素都比compile_time小 即返回列表下最后一个文件
        :param str_time: 要进行比较的时间 2020-12-12-10-07-26-935
        :param files_path: 要寻找的文件夹 2020-12-12-10-07-26-935
        :param p: 时间解析格式如
        :return:  返回最接近并且比时间str_time小的时间为文件名的文件
        """
        # 把str_time转化为可以比较的形式
        try:
            compile_t = datetime.datetime.strptime(str_time, p1)
        except Exception:
            print("需要对比的时间：%s不能被格式：%s所解析，请检查解析格式是否正确" % (str_time, p1))
            return -1
        t_list = os.listdir(files_path)
        if t_list:
            try:
                t_list.sort(key=lambda x: datetime.datetime.strptime(x.split(".")[0], p2))
            except Exception:
                print("路径下：%s文件不能被格式：%s所解析，请检查解析格式是否正确" % (files_path, p1))
            for i, tm in enumerate(t_list):
                # 获得文件所表示的时间字符串并且转换为可以比较的类型
                tm = os.path.splitext(tm)[0]
                t = datetime.datetime.strptime(tm, p2)
                if t > compile_t:
                    return t_list[i - 1] if i != 0 else -1
            return t_list[-1]
        else:
            print("路径：%s下不存在文件" % files_path)

    @staticmethod
    def findLastAfterTime(str_time, files_path=None, p1='%Y-%m-%d-%H-%M-%S-%f', p2='%Y-%m-%d-%H-%M-%S-%f'):
        """
        在files_path中寻找一个文件，这个文件名表示的时间需要比str_time大并且最接近str_time
        算法思路:
            1. 获取路径下所有文件列表 并排序
            2. 把str_time转换为可以比较的形式compile_time
            3. 遍历排序后的列表，其每个元素与compile_time进行一一对比
                1. 如果元素比str_time小继续循环
                2. 如果元素比str_time大直接返回
            4. 如果遍历完成后还没有返回 则说明所有元素都比compile_time小 即返回列表下最后一个文件
        :param str_time: 要进行比较的时间 2020-12-12-10-07-26-935
        :param files_path: 要寻找的文件夹 2020-12-12-10-07-26-935
        :param p1: 时间str_time的解析格式如
        :param p2: 文件路径files_path下文件名的时间解析格式如
        :return:  返回最接近并且比时间str_time小的时间为文件名的文件
        """
        # 把str_time转化为可以比较的形式
        try:
            compile_t = datetime.datetime.strptime(str_time, p1)
        except Exception:
            print("需要对比的时间：%s不能被格式：%s所解析，请检查解析格式是否正确" % (str_time, p1))
            return -1
        t_list = os.listdir(files_path)
        if t_list:
            try:
                t_list.sort(key=lambda x: datetime.datetime.strptime(x.split(".")[0], p2))
            except Exception:
                print("路径下：%s文件不能被格式：%s所解析，请检查解析格式是否正确" % (files_path, p1))
            for i, tm in enumerate(t_list):
                # 获得文件所表示的时间字符串并且转换为可以比较的类型
                tm = os.path.splitext(tm)[0]
                t = datetime.datetime.strptime(tm, p2)
                if t > compile_t:
                    return t_list[i]
            return t_list[-1]
        else:
            print("路径：%s下不存在文件" % files_path)

    @staticmethod
    def checkPathOrCreate(path):
        if os.path.exists(path):
            print("路径已存在->%s" % path)
        else:
            print("创建路径:%s" % path)
            os.makedirs(path, exist_ok=True)

    @staticmethod
    def moveFilesTo(start_file, end_file, src, dest, p='%Y-%m-%d-%H-%M-%S-%f'):
        """
        移动范围内文件到指定目录
        :param start_file: src下开始移动的文件名
        :param end_file: src下结束移动的文件名
        :param src: 源文件地址
        :param dest: 目的地址
        :return:
        """
        t1 = time.time()
        file_list = os.listdir(src)
        if not file_list:
            print("源文件路径[%s]为空" % src)
            return -1

        file_list.sort(key=lambda x: datetime.datetime.strptime(x.split(".")[0], p))
        start = file_list.index(start_file)
        end = file_list.index(end_file)
        for file in file_list[start:end + 1]:
            file_path = os.path.join(src, file)
            shutil.copy(file_path, dest)
        print("移动文件耗时:%s" % (time.time() - t1))

    @property
    def now(self):
        return datetime.datetime.now()

    @property
    def machineTool_output_path(self):
        """
        振动数据分类路径
        :return:
        """
        return os.path.join(self.vibrationData_move_path, self.usingTool)

    @property
    def machineOldTool_output_path(self):
        """
        振动数据分类路径
        :return:
        """
        return os.path.join(self.vibrationData_move_path, self.old_tool)

    @staticmethod
    def date_to_str(date):
        return date.strftime('%Y-%m-%d-%H-%M-%S-%f')[:-3]

    def process_vibrationData(self):
        """
        把振动数据进行变换处理

        :param file_path:  分类收振动数据存放文件路径
        :param tools_num:  当前加工刀具
        :return:
        """
        data = []
        filenames = os.listdir(self.machineTool_output_path)
        filenames.sort(key=lambda x: datetime.datetime.strptime(x.split(".")[0], settings2.VIBRATION_FILENAME_PATTERN))
        if len(filenames) < 60:
            print("路径->%s下文件数目不够" % self.machineTool_output_path)
        else:
            count = 0
            for filename in filenames[::-1]:
                if count > 60:
                    break
                data_file_path = os.path.join(self.machineTool_output_path, filename)
                with open(data_file_path, 'r') as f:
                    line = f.readline()
                    if line.endswith(","):
                        line = line[:-1]
                    if line:
                        # curLine = [x for x in line.strip().split(",") if x]
                        curLine = line.strip().split(",")
                        try:
                            fline = np.array(list(map(int, curLine)))
                        except Exception as e:
                            print(e)
                            print("文件内容有误:%s" % data_file_path)
                        tem = sqrt(np.sum(fline ** 2) / len(fline))
                        data.append(tem)
                        count += 1
            # 不够60 补齐至60个数据 极端情况下可能出现不足60个：如果文件存在但是没有内容
            for _ in range(60 - len(data)):
                data.append(data[-1])

            data = np.array(data[::-1])
            data = 1/(1+log(data.mean(), 10e12))
            d = dict(data=data)
            # en_json = json.dumps(d, ensure_ascii=True)
            now_str = self.now.strftime(settings2.OUTPUT_FILENAME_PATTERN)
            tool_name = str(self.usingTool)
            if len(tool_name) == 1:
                tool_name = "T0" + tool_name
            else:
                tool_name = "T" + tool_name
            output_file_name = "%s-%s.json" % (tool_name, now_str)
            file_path = os.path.join(self.vibration_rms_output_path, output_file_name)
            with open(file_path, 'w') as f:
                json.dump(dict(data=data), f)

    def reduce_vibrationData_fre(self):
        """
        降频振动数据原始值 把震动数据处理后移动到指定路径下
        只读取路径下最新生成的文件
        :return: Json数据文件
        """
        file_list = os.listdir(self.vibrationData_input_path)
        file_list.sort(key=lambda x: datetime.datetime.strptime(x.split(".")[0], settings2.VIBRATION_FILENAME_PATTERN))
        if file_list:
            data_path = os.path.join(self.vibrationData_input_path, file_list[-1])
            if self.machine_name == "0002":
                # print(data_path)
                pass
            with open(data_path, 'r') as f:
                line = f.readline()
                if line:
                    curLine = line.strip().split(",")

                    data = list(map(int, curLine[0:60]))

                    now_str = self.now.strftime(settings2.OUTPUT_FILENAME_PATTERN)
                    output_file_name = "%s.json" % now_str
                    file_path = os.path.join(self.virbation_raw_output_path, output_file_name)
                    with open(file_path, 'w') as f:
                        json.dump(dict(data=data), f)
            self.old_vib_data = file_list[-1]
        else:
            print("路径：%s下不存在文件" % self.vibrationData_input_path)

    def process_machineData(self):
        file_list = os.listdir(self.machineInfo_input_path)
        if file_list:
            data_path = os.path.join(self.machineInfo_input_path, file_list[-2])
            # 可能由于文件读取xlsx文件是文件正在创建过程中导致无法正确读取而报错
            try:
                df = pd.read_csv(data_path, header=None)
            except Exception as e:
                print(e)
                return
            data = np.array(df.iloc[:, 2]).tolist()
            now_str = self.now.strftime(settings2.OUTPUT_FILENAME_PATTERN)
            output_file_name = "%s.json" % now_str
            file_path = os.path.join(self.load_data_output_path, output_file_name)
            with open(file_path, 'w') as f:
                json.dump(dict(data=data), f)

        pass

    def run(self) -> None:
        """
        算法本体
        算法每隔60s运行一次指定算法程序来处理振动数据得到变换后的振动数据
        算法每隔1s运行一次指定算法程序来处理振动数据得到降频振动数据
        算法每隔1s运行一次指定算法程序来处理机台信息得到负载信息
        :return:
        """

        while 1:

            while True:
                self.reduce_vibrationData_fre()
                self.process_machineData()
                time.sleep(1)

                if not self.setToolInfo():
                    print("获取机台信息失败")
                    break
                last_trans_time_str = self.date_to_str(self.last_trans_time)
                now_str = self.date_to_str(self.now)
                print("%s->上次传输数据时间:%s, 当前时间:%s, 当前加工刀具:%s" % (
                    self.machine_name, last_trans_time_str, now_str, self.usingTool))
                if (self.now - self.last_trans_time).seconds >= 60 and self.usingTool != 0:
                    first_vib_file = self.findLastAfterTime(last_trans_time_str,
                                                            files_path=self.vibrationData_input_path,
                                                            p1="%Y-%m-%d-%H-%M-%S-%f",
                                                            p2=settings2.VIBRATION_FILENAME_PATTERN)
                    last_vib_file = self.findLastBeforTime(now_str, files_path=self.vibrationData_input_path)
                    self.checkPathOrCreate(self.machineTool_output_path)
                    self.last_trans_time = self.now
                    self.moveFilesTo(first_vib_file, last_vib_file, self.vibrationData_input_path,
                                     self.machineTool_output_path)

                    t1 = time.time()
                    self.process_vibrationData()
                    print("处理完成时间：%s" % (time.time() - t1))
                # 如果换刀了 需要移动
                if self.hasChangeTool:
                    first_vib_file = self.findLastAfterTime(last_trans_time_str,
                                                            files_path=self.vibrationData_input_path,
                                                            p1="%Y-%m-%d-%H-%M-%S-%f",
                                                            p2=settings2.VIBRATION_FILENAME_PATTERN)
                    last_vib_file = self.findLastBeforTime(now_str, files_path=self.vibrationData_input_path,
                                                           p1="%Y-%m-%d-%H-%M-%S-%f",
                                                           p2=settings2.VIBRATION_FILENAME_PATTERN
                                                           )
                    self.checkPathOrCreate(self.machineOldTool_output_path)
                    self.last_trans_time = self.now
                    self.moveFilesTo(first_vib_file, last_vib_file, self.vibrationData_input_path,
                                     self.machineOldTool_output_path)

if __name__ == '__main__':
    try:

        t_list = []
        for machine in settings2.MACHINE_SETTINGS:
            t_list.append(
                ProcessFile(machine_name=machine["param"], machineInfo_input_path=machine["MachineData_INPUT_PATH"],
                            vibrationData_input_path=machine["SensorData_INPUT_PATH"],
                            using_tool_input_path=machine['ToolInfo_INPUT_PATH'],
                            vibrationData_move_path=machine["SensorData_MOVE_PATH"],
                            result_output_path=machine["Result_OUPUT_PATH"]))
            t_list.append(
                threading.Thread(target=machineDataSrcSuperviser, args=(180, machine["MachineData_INPUT_PATH"])))
            t_list.append(threading.Thread(target=keepRMSData, args=(
                os.path.join(machine["Result_OUPUT_PATH"], settings2.VIRBATION_RMS_OUTPUT_PATH_NAME),
                settings2.RMS_RESULT_RANGE)))
        t_list.append(threading.Thread(target=machineDataSuperviser, args=(120,)))
        t_list.append(threading.Thread(target=resultDataSuperviser, args=(120,)))

        for t in t_list:
            t.start()
        for t in t_list:
            t.join()
    except Exception as e:
        print(e)


import scrapy
import logging
from covid19.items import TestingStats
import requests
import json
from datetime import datetime as dt
from dateutil.parser import parse

class NewMexicoSpider( scrapy.Spider ) :

    name = "newmexico"
    allowed_domains = ["https://cv.nmhealth.org/"]
    obj = ["NewMexico"]
    case_categories = ["positive", "negative", "Pending"]
    names = ["New Mexico" ]
    custom_settings = { "LOG_LEVEL" : logging.ERROR }

    def start_requests( self ):
        yield scrapy.Request( "https://cv.nmhealth.org/", callback=self.parse )

    def parse( self, response ):
        item = TestingStats()
        positive = response.xpath( "/html/body/div/div[2]/div/article/div/div/div/div/div[2]/div/div[2]/div[2]/div/table/tbody/tr[1]/td[2]/text()" ).get()
        negative = response.xpath( "/html/body/div/div[2]/div/article/div/div/div/div/div[2]/div/div[2]/div[2]/div/table/tbody/tr[2]/td[2]/text()" ).get()
        total = response.xpath( "/html/body/div/div[2]/div/article/div/div/div/div/div[2]/div/div[2]/div[2]/div/table/tbody/tr[3]/td[2]/text()" ).get()
        positive = positive.replace( ",", "" )
        negative = negative.replace(",", "")
        total = total.replace(",", "")
        positive = positive
        negative = negative
        total = total

        date = response.xpath( '/html/body/div/div[2]/div/article/div/div/div/div/div[2]/div/div[2]/div[2]/div/p[3]/em/text()' ).get()
        date = parse( date, fuzzy=True )

        item["date"] = date.strftime( "%Y-%m-%d %H:%M %p" )
        item["name"] = self.names[0]
        item["positive"] = positive
        item["negative"] = negative
        item["pending"] = int( total ) - ( int( positive ) + int( negative ) )

        print( item.toAsciiTable() )
        return item


import pytest
import numpy as np
import pandas as pd
from inference.inference import Infer

data = pd.DataFrame()

class TestInfer:

    def test_from_two_arrays(self):
        test_infer = Infer(data)
        ar1 = np.random.rand(1, 10)
        ar2 = np.random.rand(1, 10)
        df = test_infer.from_two_arrays(ar1, ar2)
        value_in_df = "value" in df
        id_in_df = "group_id" in df
        assert value_in_df is True, id_in_df is True

    def test_from_two_proportions(self):
        test_infer = Infer(data)
        prop_1 = 75
        prop_2 = 25
        len_data1 = len_data2 = 10
        df = test_infer.from_two_proportions(prop_1, prop_2, len_data1, len_data2)
        value_in_df = "value" in df
        id_in_df = "group_id" in df
        assert value_in_df is True, id_in_df is True

    def test_test_independence(self):
        pass


import logging
import os
import pickle

import numpy as np

from training.xgboost_train import xgboost_data_preparation_to_predict

logger = logging.getLogger(__name__)
formatting = (
    "%(asctime)s: %(levelname)s: File:%(filename)s Function:%(funcName)s Line:%(lineno)d "
    "message:%(message)s"
)
logging.basicConfig(
    filename=os.path.join(os.path.dirname(os.path.dirname(__file__)), "logs/logs.log"),
    level=logging.DEBUG,
    format=formatting,
)

def predict_unseen_data(predict: dict, models_nr: list, save_models_dir: str, model_type: str) -> np.array:
    """ Target prediction

    This function predicts unseen data given in the dictionary `predict`

    :param dict predict: A dictionary that contains pandas dataframes which are unseen data that will be used to
            predict the target
    :param list models_nr: A list of indexes that will be used to point to the trained models which were saved
            locally after training.
    :param str save_models_dir: The path where the models will be saved.
    :param str model_type: The type of the model that was used to fit the data e.g. linear or lightgbm

    :return:
            - y_prediction - A numpy array that contains the predicted target of the unseen data.
    """

    y_prediction = 0
    model = None

    for data_key, dataframe in predict.items():
        if model_type == "xgboost":
            dataframe_i = predict[data_key]["features"]
            dataframe_i.columns = [x for x in range(len(list(dataframe_i.columns)))]
            array = xgboost_data_preparation_to_predict(dataframe_i)
        else:
            array = predict[data_key]["features"].values

        # reset the variable
        y_prediction = 0

        for model_i in models_nr:
            try:
                with open(os.path.join(save_models_dir, f"{model_type}_{model_i}.pkl"), "rb") as save_model:
                    model = pickle.load(save_model)
                logger.info("The model is loaded successfully: " + f"{model_type}_{model_i}.pkl")
            except Exception as e:
                logger.error(f"Loading the model failed. Error is {e}")

            y_prediction += model.predict(array)

        # Stacking the results of different models using the mean value
        y_prediction = y_prediction / len(models_nr)

    return y_prediction

def model_prediction(predict: dict, models_nr: list, save_models_dir: str, model_type: str) -> np.array:
    """Predictor

    This function applies the function `predict_unseen_data` to predict the target using the data in the `predict`.

    :param dict predict: A dictionary that contains pandas dataframes which are unseen data that will be used to
            predict the target
    :param list models_nr: A list of indexes that will be used to point to the trained models which were saved
            locally after training.
    :param str save_models_dir: The path where the models will be saved.
    :param str model_type: The type of the model that was used to fit the data e.g. linear or lightgbm

    :return:
            - y_prediction - A numpy array that contains the predicted target of the unseen data.
    """

    # Predict the target of unseen data
    y_prediction = 0

    if isinstance(predict, bool):
        logger.info("Nothing will be predicted")
    else:
        try:
            y_prediction = predict_unseen_data(predict, models_nr, save_models_dir, model_type)
            logger.info("Prediction process is finished!")
        except Exception as e:
            logger.error(f"Prediction failed: The error is {e}")

    return y_prediction


from src.SpectralIndex import SpectralIndex
from typing import Dict
from pydantic import BaseModel

class SpectralIndices(BaseModel):
    SpectralIndices: Dict[str, SpectralIndex]

spindex = SpectralIndices(
    SpectralIndices=dict(
        BNDVI=SpectralIndex(
            short_name='BNDVI',
            long_name='Blue Normalized Difference Vegetation Index',
            formula='(N - B)/(N + B)',
            reference='https://www.indexdatabase.de/db/i-single.php?id=135',
            type='vegetation',
            date_of_addition='2021-04-07',
            contributor="https://github.com/davemlz"
        ),
        CIG=SpectralIndex(
            short_name='CIG',
            long_name='Chlorophyll Index Green',
            formula='(N / G) - 1.0',
            reference='https://www.indexdatabase.de/db/i-single.php?id=391',
            type='vegetation',
            date_of_addition='2021-04-07',
            contributor="https://github.com/davemlz"
        ),
        CVI=SpectralIndex(
            short_name='CVI',
            long_name='Chlorophyll Vegetation Index',
            formula='(N * R) / (G ** 2.0)',
            reference='https://www.indexdatabase.de/db/i-single.php?id=391',
            type='vegetation',
            date_of_addition='2021-04-07',
            contributor="https://github.com/davemlz"
        ),
        EVI=SpectralIndex(
            short_name='EVI',
            long_name='Enhanced Vegetation Index',
            formula='g * (N - R) / (N + C1 * R - C2 * B + L)',
            reference='https://www.indexdatabase.de/db/i-single.php?id=16',
            type='vegetation',
            date_of_addition='2021-04-07',
            contributor="https://github.com/davemlz"
        ),
        EVI2=SpectralIndex(
            short_name='EVI2',
            long_name='Two-Band Enhanced Vegetation Index',
            formula='g * (N - R) / (N + 2.4 * R + L)',
            reference='https://doi.org/10.1016/j.rse.2008.06.006',
            type='vegetation',
            date_of_addition='2021-04-07',
            contributor="https://github.com/davemlz"
        ),
        GARI=SpectralIndex(
            short_name='GARI',
            long_name='Green Atmospherically Resistant Vegetation Index',
            formula='(N - (G - (B - R))) / (N - (G + (B - R)))',
            reference='https://www.indexdatabase.de/db/i-single.php?id=363',
            type='vegetation',
            date_of_addition='2021-04-07',
            contributor="https://github.com/davemlz"
        ),
        GBNDVI=SpectralIndex(
            short_name='GBNDVI',
            long_name='Green-Blue Normalized Difference Vegetation Index',
            formula='(N - (G + B))/(N + (G + B))',
            reference='https://www.indexdatabase.de/db/i-single.php?id=186',
            type='vegetation',
            date_of_addition='2021-04-07',
            contributor="https://github.com/davemlz"
        ),
        GEMI=SpectralIndex(
            short_name='GEMI',
            long_name='Global Environment Monitoring Index',
            formula='((2.0*((N ** 2.0)-(R ** 2.0)) + 1.5*N + 0.5*R)/(N + R + 0.5))*(1.0 - 0.25*((2.0 * ((N ** 2.0) - (R ** 2)) + 1.5 * N + 0.5 * R)/(N + R + 0.5)))-((R - 0.125)/(1 - R))',
            reference='https://www.indexdatabase.de/db/i-single.php?id=25',
            type='vegetation',
            date_of_addition='2021-04-07',
            contributor="https://github.com/davemlz"
        ),
        GLI=SpectralIndex(
            short_name='GLI',
            long_name='Green Leaf Index',
            formula='(2.0 * G - R - B) / (2.0 * G + R + B)',
            reference='https://www.indexdatabase.de/db/i-single.php?id=375',
            type='vegetation',
            date_of_addition='2021-04-07',
            contributor="https://github.com/davemlz"
        ),
        GNDVI=SpectralIndex(
            short_name='GNDVI',
            long_name='Green Normalized Difference Vegetation Index',
            formula='(N - G)/(N + G)',
            reference='https://www.indexdatabase.de/db/i-single.php?id=401',
            type='vegetation',
            date_of_addition='2021-04-07',
            contributor="https://github.com/davemlz"
        ),
        GRNDVI=SpectralIndex(
            short_name='GRNDVI',
            long_name='Green-Red Normalized Difference Vegetation Index',
            formula='(N - (G + R))/(N + (G + R))',
            reference='https://www.indexdatabase.de/db/i-single.php?id=185',
            type='vegetation',
            date_of_addition='2021-04-07',
            contributor="https://github.com/davemlz"
        ),
        GVMI=SpectralIndex(
            short_name='GVMI',
            long_name='Global Vegetation Moisture Index',
            formula='((N + 0.1) - (S2 + 0.02)) / ((N + 0.1) + (S2 + 0.02))',
            reference='https://www.indexdatabase.de/db/i-single.php?id=372',
            type='vegetation',
            date_of_addition='2021-04-07',
            contributor="https://github.com/davemlz"
        ),
        MNDVI=SpectralIndex(
            short_name='MNDVI',
            long_name='Modified Normalized Difference Vegetation Index',
            formula='(N - S2)/(N + S2)',
            reference='https://www.indexdatabase.de/db/i-single.php?id=245',
            type='vegetation',
            date_of_addition='2021-04-07',
            contributor="https://github.com/davemlz"
        ),
        NDVI=SpectralIndex(
            short_name='NDVI',
            long_name='Normalized Difference Vegetation Index',
            formula='(N - R)/(N + R)',
            reference='https://www.indexdatabase.de/db/i-single.php?id=58',
            type='vegetation',
            date_of_addition='2021-04-07',
            contributor="https://github.com/davemlz"
        ),
        NGRDI=SpectralIndex(
            short_name='NGRDI',
            long_name='Normalized Green Red Difference Index',
            formula='(G - R) / (G + R)',
            reference='https://www.indexdatabase.de/db/i-single.php?id=390',
            type='vegetation',
            date_of_addition='2021-04-07',
            contributor="https://github.com/davemlz"
        ),
        RVI=SpectralIndex(
            short_name='RVI',
            long_name='Ratio Vegetation Index',
            formula='N / R',
            reference='https://www.indexdatabase.de/db/i-single.php?id=72',
            type='vegetation',
            date_of_addition='2021-04-07',
            contributor="https://github.com/davemlz"
        ),
        SAVI=SpectralIndex(
            short_name='SAVI',
            long_name='Soil-Adjusted Vegetation Index',
            formula='(1.0 + L) * (N - R) / (N + R + L)',
            reference='https://www.indexdatabase.de/db/i-single.php?id=87',
            type='vegetation',
            date_of_addition='2021-04-07',
            contributor="https://github.com/davemlz"
        ),
        VARI=SpectralIndex(
            short_name='VARI',
            long_name='Visible Atmospherically Resistant Index',
            formula='(G - R) / (G + R - B)',
            reference='https://www.indexdatabase.de/db/i-single.php?id=356',
            type='vegetation',
            date_of_addition='2021-04-07',
            contributor="https://github.com/davemlz"
        ),
        BAI=SpectralIndex(
            short_name='BAI',
            long_name='Burned Area Index',
            formula='1.0 / ((0.1 - R) ** 2.0 + (0.06 - N) ** 2.0)',
            reference='https://digital.csic.es/bitstream/10261/6426/1/Martin_Isabel_Serie_Geografica.pdf',
            type='burn',
            date_of_addition='2021-04-07',
            contributor="https://github.com/davemlz"
        ),
        BAIS2=SpectralIndex(
            short_name='BAIS2',
            long_name='Burned Area Index for Sentinel 2',
            formula='(1.0 - ((RE2 * RE3 * RE4) / R) ** 0.5) * (((S2 - RE4)/(S2 + RE4) ** 0.5) + 1.0)',
            reference='https://doi.org/10.3390/ecrs-2-05177',
            type='burn',
            date_of_addition='2021-04-07',
            contributor="https://github.com/davemlz"
        ),
        CSIT=SpectralIndex(
            short_name='CSIT',
            long_name='Char Soil Index Thermal',
            formula='N / (S2 * T1 / 10000.0)',
            reference='https://doi.org/10.1080/01431160600954704',
            type='burn',
            date_of_addition='2021-04-07',
            contributor="https://github.com/davemlz"
        ),
        NBR=SpectralIndex(
            short_name='NBR',
            long_name='Normalized Burn Ratio',
            formula='(N - S2) / (N + S2)',
            reference='https://www.indexdatabase.de/db/i-single.php?id=53',
            type='burn',
            date_of_addition='2021-04-07',
            contributor="https://github.com/davemlz"
        ),
        NBRT=SpectralIndex(
            short_name='NBRT',
            long_name='Normalized Burn Ratio Thermal',
            formula='(N - (S2 * T1 / 10000.0)) / (N + (S2 * T1 / 10000.0))',
            reference='https://doi.org/10.1080/01431160500239008',
            type='burn',
            date_of_addition='2021-04-07',
            contributor="https://github.com/davemlz"
        ),
        NDVIT=SpectralIndex(
            short_name='NDVIT',
            long_name='Normalized Difference Vegetation Index Thermal',
            formula='(N - (R * T1 / 10000.0))/(N + (R * T1 / 10000.0))',
            reference='https://doi.org/10.1080/01431160600954704',
            type='burn',
            date_of_addition='2021-04-07',
            contributor="https://github.com/davemlz"
        ),
        SAVIT=SpectralIndex(
            short_name='SAVIT',
            long_name='Soil-Adjusted Vegetation Index Thermal',
            formula='(1.0 + L) * (N - (R * T1 / 10000.0)) / (N + (R * T1 / 10000.0) + L)',
            reference='https://doi.org/10.1080/01431160600954704',
            type='burn',
            date_of_addition='2021-04-07',
            contributor="https://github.com/davemlz"
        ),
        MNDWI=SpectralIndex(
            short_name='MNDWI',
            long_name='Modified Normalized Difference Water Index',
            formula='(G - S1) / (G + S1)',
            reference='https://doi.org/10.1080/01431160600589179',
            type='water',
            date_of_addition='2021-04-07',
            contributor="https://github.com/davemlz"
        ),
        NDWI=SpectralIndex(
            short_name='NDWI',
            long_name='Normalized Difference Water Index',
            formula='(G - N) / (G + N)',
            reference='https://doi.org/10.1080/01431169608948714',
            type='water',
            date_of_addition='2021-04-07',
            contributor="https://github.com/davemlz"
        ),
        NDSI=SpectralIndex(
            short_name='NDSI',
            long_name='Normalized Difference Snow Index',
            formula='(G - S1) / (G + S1)',
            reference='https://doi.org/10.1109/IGARSS.1994.399618',
            type='snow',
            date_of_addition='2021-04-07',
            contributor="https://github.com/davemlz"
        ),
        NDDI=SpectralIndex(
            short_name='NDDI',
            long_name='Normalized Difference Drought Index',
            formula='(((N - R)/(N + R)) - ((G - N)/(G + N)))/(((N - R)/(N + R)) + ((G - N)/(G + N)))',
            reference='https://doi.org/10.1029/2006GL029127',
            type='drought',
            date_of_addition='2021-04-07',
            contributor="https://github.com/davemlz"
        ),
        kEVI=SpectralIndex(
            short_name='kEVI',
            long_name='Kernel Enhanced Vegetation Index',
            formula='g * (kNN - kNR) / (kNN + C1 * kNR - C2 * kNB + kNL)',
            reference='https://doi.org/10.1126/sciadv.abc7447',
            type='kernel',
            date_of_addition='2021-04-07',
            contributor="https://github.com/davemlz"
        ),
        kNDVI=SpectralIndex(
            short_name='kNDVI',
            long_name='Kernel Normalized Difference Vegetation Index',
            formula='(kNN - kNR)/(kNN + kNR)',
            reference='https://doi.org/10.1126/sciadv.abc7447',
            type='kernel',
            date_of_addition='2021-04-07',
            contributor="https://github.com/davemlz"
        ),
        kRVI=SpectralIndex(
            short_name='kRVI',
            long_name='Kernel Ratio Vegetation Index',
            formula='kNN / kNR',
            reference='https://doi.org/10.1126/sciadv.abc7447',
            type='kernel',
            date_of_addition='2021-04-07',
            contributor="https://github.com/davemlz"
        ),
        kVARI=SpectralIndex(
            short_name='kVARI',
            long_name='Kernel Visible Atmospherically Resistant Index',
            formula='(kGG - kGR) / (kGG + kGR - kGB)',
            reference='https://doi.org/10.1126/sciadv.abc7447',
            type='kernel',
            date_of_addition='2021-04-07',
            contributor="https://github.com/davemlz"
        ),
        SeLI=SpectralIndex(
            short_name='SeLI',
            long_name='Sentinel-2 LAI Green Index',
            formula='(RE4 - RE1) / (RE4 + RE1)',
            reference='https://doi.org/10.3390/s19040904',
            type='vegetation',
            date_of_addition='2021-04-08',
            contributor="https://github.com/davemlz"
        )
    )
)


import inline as inline
import matplotlib
import numpy as np

#%matplotlib inline

from matplotlib import pyplot as plt

# набор данных ( необходимо для работы sgd)
X = np.array([
    [-2, 4, -1],
    [4, 1, -1],
    [1, 6, -1],
    [2, 4, -1],
    [6, 2, -1],

])

y = np.array([-1, -1, 1, 1, 1])

# стохастический градиентынй спуск
def perceptron_sgd(X, Y):
    w = np.zeros(len(X[0]))     # инициализация вектора весов
    eta = 1                     # скорость обучения
    epochs = 20                 # количсество эпох

    for t in range(epochs):                    # итерация n раз
        for i, x in enumerate(X):              # итерация по каждой выборке
            if (np.dot(X[i], w) * Y[i]) <= 0:  # условие неправильности классификации
                w = w + eta * X[i] * Y[i]      # обновление правила для весов

    return w

w = perceptron_sgd(X, y)
print(w)

for d, sample in enumerate(X):

    if d < 2:
        plt.scatter(sample[0], sample[1], s=120, marker='_', linewidths=2)

    else:
        plt.scatter(sample[0], sample[1], s=120, marker='+', linewidths=2)

plt.plot([-2, 6], [6, 0.5])     # Построить график
plt.show()                      # Отобразить график

# Учится (2 график)
def perceptron_sgd_plot(X, Y):
    '''
    train perceptron and plot the total loss in each epoch.

    :param X: data samples
    :param Y: data labels
    :return: weight vector as a numpy array
    '''
    w = np.zeros(len(X[0]))
    eta = 1
    n = 30
    errors = []

    for t in range(n):
        total_error = 0
        for i, x in enumerate(X):
            if (np.dot(X[i], w) * Y[i]) <= 0:
                total_error += (np.dot(X[i], w) * Y[i])
                w = w + eta * X[i] * Y[i]
        errors.append(total_error * -1)  # подсчет ошибок в каждой эпохе

    plt.plot(errors)
    plt.xlabel('Epoch')
    plt.ylabel('Total Loss')
    plt.show()

    return w

print(perceptron_sgd_plot(X, y))

# Evaluation
for d, sample in enumerate(X):
    # Plot the negative samples
    if d < 2:
        plt.scatter(sample[0], sample[1], s=120, marker='_', linewidths=2)
    # Plot the positive samples
    else:
        plt.scatter(sample[0], sample[1], s=120, marker='+', linewidths=2)

# добавить тестовые значения
plt.scatter(2, 2, s=120, marker='_', linewidths=2, color='yellow')
plt.scatter(4, 3, s=120, marker='+', linewidths=2, color='blue')

# Print the hyperplane calculated by perceptron_sgd()
x2 = [w[0], w[1], -w[1], w[0]]
x3 = [w[0], w[1], w[1], -w[0]]

x2x3 = np.array([x2, x3])
X, Y, U, V = zip(*x2x3)
ax = plt.gca()
ax.quiver(X, Y, U, V, scale=1, color='blue')

plt.show()


from django import forms
from django.forms import ModelForm
from django.utils.translation import ugettext_lazy as _
from django.contrib.auth import get_user_model

from .models import Invitation
from .adapters import get_invitations_adapter
from .exceptions import AlreadyInvited, AlreadyAccepted, UserRegisteredEmail

class CleanEmailMixin(object):

    def validate_invitation(self, email):
        if Invitation.objects.all_valid().filter(
                email__iexact=email, accepted=False):
            raise AlreadyInvited
        elif Invitation.objects.filter(
                email__iexact=email, accepted=True):
            raise AlreadyAccepted
        elif get_user_model().objects.filter(email__iexact=email):
            raise UserRegisteredEmail
        else:
            return True

    def clean_email(self):
        email = self.cleaned_data["email"]
        email = get_invitations_adapter().clean_email(email)

        errors = {
            "already_invited": _("This e-mail address has already been"
                                 " invited."),
            "already_accepted": _("This e-mail address has already"
                                  " accepted an invite."),
            "email_in_use": _("An active user is using this e-mail address"),
        }
        try:
            self.validate_invitation(email)
        except(AlreadyInvited):
            raise forms.ValidationError(errors["already_invited"])
        except(AlreadyAccepted):
            raise forms.ValidationError(errors["already_accepted"])
        except(UserRegisteredEmail):
            raise forms.ValidationError(errors["email_in_use"])
        return email

class InviteForm(ModelForm):
    class Meta:
        model = Invitation
        fields = ['email', 'group']

class InvitationAdminAddForm(forms.ModelForm, CleanEmailMixin):
    email = forms.EmailField(
        label=_("E-mail"),
        required=True,
        widget=forms.TextInput(attrs={"type": "email", "size": "30"}))
    group = forms.CharField(label="Group")

    def save(self, *args, **kwargs):
        cleaned_data = super(InvitationAdminAddForm, self).clean()
        email = cleaned_data.get("email")
        params = {'email': email}
        if cleaned_data.get("inviter"):
            params['inviter'] = cleaned_data.get("inviter")
        instance = Invitation.create(**params)
        instance.send_invitation(self.request)
        super(InvitationAdminAddForm, self).save(*args, **kwargs)
        return instance

    class Meta:
        model = Invitation
        fields = ("email", "inviter")

class InvitationAdminChangeForm(forms.ModelForm):

    class Meta:
        model = Invitation
        fields = '__all__'