# -*- coding: utf-8 -*-
'''
Задание 19.3b

Дополнить функцию send_commands таким образом, чтобы перед подключением к устройствам по SSH,
выполнялась проверка доступности устройства pingом (можно вызвать команду ping в ОС).

> Как выполнять команды ОС, описано в разделе 11_modules/subprocess.html. Там же есть пример функции с отправкой ping.

Если устройство доступно, можно выполнять подключение.
Если не доступно, вывести сообщение о том, что устройство с определенным IP-адресом недоступно
и не выполнять подключение  к этому устройству.

Для удобства можно сделать отдельную функцию для проверки доступности
и затем использовать ее в функции send_commands.
'''

import netmiko
import yaml
from netmiko import ConnectHandler
import subprocess

commands = [
    'logging 10.255.255.1', 'logging buffered 20010', 'no logging console'
]
command = 'sh ip int br'

def send_commands(device, show=False, filename=False, config=False):
    if show:
        zaloopec = send_show_command(p, command)
        print(zaloopec)
    elif filename:
        zaloopec = send_from_filename(p, 'config.txt')
        print(zaloopec)
    elif config:
        zaloopec = send_config_command(p, commands)
        print(zaloopec)

with open('devices.yaml') as dev:
    devs_param = yaml.load(dev)
    for zzz in devs_param['routers']:
        #print(zzz)
        p = devs_param['routers']

def check_ping(device_to_check):
    reply = subprocess.run(['ping', '-c', '3', '-n', device_to_check])
    if reply.returncode == False:
        return True
    else:
        return False

def send_show_command(dev_to_send, commands_to_send):
    rslt = {}
    for each in dev_to_send:
        #print(each)
        zopka = check_ping(each['ip'])
        #print(zopka)
        if zopka:
            try:
                with ConnectHandler(**each) as ssh:
                    ssh.enable()
                    result = ssh.send_command(commands_to_send)
                    tmp_ip = each['ip']
                    rslt[tmp_ip] = result
            except netmiko.ssh_exception.NetMikoAuthenticationException:
                #incorrect credentials
                print('Authentication failed!')
                continue
            except netmiko.ssh_exception.NetMikoTimeoutException:
                #wrong IP address
                print('Connection Timeout for {} host!'.format(each['ip']))
                continue
        else:
            continue
    return rslt

def send_config_command(dev_to_send, commands_to_send):
    rslt = {}
    for each in dev_to_send:
        zopka = check_ping(each['ip'])
        #print(each)
        if zopka:
            try:
                with ConnectHandler(**each) as ssh:
                    for pizda in commands_to_send:
                        ssh.enable()
                        result = ssh.send_config_set(pizda)
                        tmp_ip = each['ip']
                        rslt[tmp_ip] = result
            except netmiko.ssh_exception.NetMikoAuthenticationException:
                #incorrect credentials
                print('Authentication failed!')
                continue
            except netmiko.ssh_exception.NetMikoTimeoutException:
                #wrong IP address
                print('Connection Timeout for {} host!'.format(each['ip']))
                continue
        else:
            continue
        return rslt

def send_from_filename(dev_to_send, filenm):
    rslt = {}
    for each in dev_to_send:
        with open(filenm) as f:
            try:
                with ConnectHandler(**each) as ssh:
                    ssh.enable()
                    #print(each)
                    for pepiska in f:
                        result = ssh.send_config_set(pepiska)
                        tmp_ip = each['ip']
                        rslt[tmp_ip] = result
                        print(rslt)
            except netmiko.ssh_exception.NetMikoAuthenticationException:
                #incorrect credentials
                print('Authentication failed!')
                continue
            except netmiko.ssh_exception.NetMikoTimeoutException:
                #wrong IP address
                print('Connection Timeout for {} host!'.format(each['ip']))
                continue
    return rslt

send_commands(p, filename=True)


# -*- coding: utf-8 -*-

import requests
import shutil
import os
from api.base.utils import waterbutler_api_url_for
from website import settings

def download_file(osf_cookie, file_node, download_path, **kwargs):
    '''
    Download an waterbutler file by streaming its contents while saving,
    so we do not waste memory.
    '''
    download_filename = file_node.name
    if not download_filename:
        download_filename = os.path.basename(file_node.path)
    assert download_filename

    full_path = os.path.join(download_path, download_filename)
    response = requests.get(
        file_node.generate_waterbutler_url(action='download', direct=None, **kwargs),
        cookies={settings.COOKIE_NAME: osf_cookie},
        stream=True
    )

    with open(full_path, 'wb') as f:
        shutil.copyfileobj(response.raw, f)

    response.close()
    return full_path

def upload_folder_recursive(osf_cookie, pid, local_path, dest_path):
    '''
    Upload all the content (files and folders) inside a folder.
    '''
    count = {
        'fail_file': 0,
        'fail_folder': 0
    }
    content_list = os.listdir(local_path)

    for item_name in content_list:
        full_path = os.path.join(local_path, item_name)

        # Upload folder
        if os.path.isdir(full_path):
            folder_response = create_folder(osf_cookie, pid, item_name, dest_path)
            if folder_response.status_code == requests.codes.created:
                folder_id = folder_response.json()['data']['id']
                rec_response = upload_folder_recursive(osf_cookie, pid, full_path, folder_id)
                count['fail_file'] += rec_response['fail_file']
                count['fail_folder'] += rec_response['fail_folder']
            else:
                count['fail_folder'] += 1

        # Upload file
        else:
            file_response = upload_file(osf_cookie, pid, full_path, item_name, dest_path)
            if file_response.status_code != requests.codes.created:
                count['fail_file'] += 1

    return count

def create_folder(osf_cookie, pid, folder_name, dest_path):
    dest_arr = dest_path.split('/')
    response = requests.put(
        waterbutler_api_url_for(
            pid, dest_arr[0], path='/' + os.path.join(*dest_arr[1:]),
            name=folder_name, kind='folder', meta='', _internal=True
        ),
        cookies={
            'osf': osf_cookie
        }
    )
    return response

def upload_file(osf_cookie, pid, file_path, file_name, dest_path):
    response = None
    dest_arr = dest_path.split('/')
    with open(file_path, 'r') as f:
        response = requests.put(
            waterbutler_api_url_for(
                pid, dest_arr[0], path='/' + os.path.join(*dest_arr[1:]),
                name=file_name, kind='file', _internal=True
            ),
            data=f,
            cookies={
                'osf': osf_cookie
            }
        )
    return response


import pathlib
from typing import Optional

import numpy as np
from einops import rearrange
from numpy import ndarray

from .._default_backends import _default_image_io_backend
from ...backend.visual_lib import VisualLib
from ...util.type import T_ARRAY

class ImageWriter:
    """ImageWriter for saving images from :class:`~torch.Tensor` or :class:`~numpy.ndarray`"""

    @classmethod
    def _convert_img_format(cls, img_in: T_ARRAY, backend: VisualLib, channel_first: bool):
        # torchvision requires (C, H, W) but others requires (H, W, C), normalize the shape
        if backend == VisualLib.PYTORCH and not channel_first:
            img_in = rearrange(img_in, 'H W C -> C H W')
        elif backend != VisualLib.PYTORCH and channel_first:
            img_in = rearrange(img_in, 'C H W -> H W C')

        # convert value range to [0, 255] int
        if isinstance(img_in, ndarray):
            img_out = (img_in * 255).astype(np.uint8)
        else:
            try:
                import torch
                if isinstance(img_in, torch.Tensor):
                    img_out = (img_in * 255).type(torch.IntTensor)
                else:
                    raise ValueError("Unknown data type. The image array type must be numpy.ndarray or torch.Tensor.")
            except ImportError:
                raise ValueError("Unknown data type. The image array type must be numpy.ndarray or torch.Tensor.")

        return img_out

    @classmethod
    def to_jpeg(cls, path: str, image: T_ARRAY, quality: int = 75, channel_first: bool = False,
        backend: Optional[VisualLib] = None
    ) -> None:
        """
        Save as jpeg files from :class:`~torch.Tensor` or :class:`~numpy.ndarray`. The value range is [0, 1].

        Args:
            path (``str``): The path of output file.
            image (:class:`~torch.Tensor` | :class:`~numpy.ndarray`): The image tensor for output.
            quality (``int``, optional): The quality level for jpg from 0 to 100. Higher means quality is better.
                Default: 75.
            channel_first (``bool``, optional): The flag for channel first.
                Specify the image shape is (H, W, C) if False, else (C, H, W) if True Default: False.
            backend (:class:`~tensorneko_util.backend.visual_lib.VisualLib`, optional): The backend library for saving.
                Default: "opencv" if installed else "matplotlib".
        """
        backend = backend or _default_image_io_backend()
        image = cls._convert_img_format(image, backend, channel_first)

        if backend == VisualLib.OPENCV:
            import cv2
            image = cv2.cvtColor(image, cv2.COLOR_RGB2BGR)
            cv2.imwrite(path, image, [cv2.IMWRITE_JPEG_QUALITY, quality])
        elif backend == VisualLib.MATPLOTLIB:
            import matplotlib.pyplot as plt
            plt.imsave(path, image, format='jpeg')
        elif backend == VisualLib.PIL:
            import PIL.Image
            PIL.Image.fromarray(image).save(path, format='jpeg', quality=quality)
        elif backend == VisualLib.PYTORCH:
            import torchvision.io
            torchvision.io.write_jpeg(image, path, quality=quality)
        else:
            raise ValueError("Unknown backend library.")

    @classmethod
    def to_png(cls, path: str, image: T_ARRAY, compression_level: int = 6, channel_first: bool = False,
        backend: Optional[VisualLib] = None
    ) -> None:
        """
        Save as png files from :class:`~torch.Tensor` or :class:`~numpy.ndarray` with (C, H, W).

        Args:
            path (``str``): The path of output file.
            image (:class:`~torch.Tensor` | :class:`~numpy.ndarray`): The image tensor for output.
            compression_level (``int``, optional): The compression level png from 0 to 9. Higher means quality is worse.
                Default: 6
            channel_first (``bool``, optional): The flag for channel first.
                Specify the image shape is (H, W, C) if False, else (C, H, W) if True Default: False.
            backend (:class:`~tensorneko_util.backend.visual_lib.VisualLib`, optional): The backend library for saving.
                Default: "opencv" if installed else "matplotlib".
        """
        backend = backend or _default_image_io_backend()
        image = cls._convert_img_format(image, backend, channel_first)
        if backend == VisualLib.OPENCV:
            if not VisualLib.opencv_available():
                raise ValueError("OpenCV is not installed.")
            import cv2
            image = cv2.cvtColor(image, cv2.COLOR_RGB2BGR)
            cv2.imwrite(path, image, [cv2.IMWRITE_PNG_COMPRESSION, compression_level])
        elif backend == VisualLib.MATPLOTLIB:
            if not VisualLib.matplotlib_available():
                raise ValueError("Matplotlib is not installed.")
            import matplotlib.pyplot as plt
            plt.imsave(path, image, format='png')
        elif backend == VisualLib.PIL:
            if not VisualLib.pil_available():
                raise ValueError("PIL is not installed.")
            import PIL.Image
            PIL.Image.fromarray(image).save(path, format='png', compress_level=compression_level)
        elif backend == VisualLib.PYTORCH:
            if not VisualLib.pytorch_available():
                raise ValueError("Torchvision is not installed.")
            import torchvision.io
            torchvision.io.write_png(image, path, compression_level=compression_level)
        else:
            raise ValueError("Unknown backend library.")

    @classmethod
    def to(cls, path: str, image: T_ARRAY, *args, **kwargs) -> None:
        """
        Save as png files from :class:`~torch.Tensor` or :class:`~numpy.ndarray` with (C, H, W).

        Args:
            path (``str``): The path of output file.
            image (:class:`~torch.Tensor` | :class:`~numpy.ndarray`): The image tensor for output.
            *args: The arguments for :meth:`ImageWriter.to_jpeg` or :meth:`ImageWriter.to_png`.
            **kwargs: The keyword arguments for :meth:`ImageWriter.to_jpeg` or :meth:`ImageWriter.to_png`.
        """
        ext = pathlib.Path(path).suffix
        if ext in ('.jpg', '.jpeg'):
            cls.to_jpeg(path, image, *args, **kwargs)
        elif ext == '.png':
            cls.to_png(path, image, *args, **kwargs)
        else:
            raise ValueError("Unknown file extension. Now only support .jpg, .jpeg and .png.")

    def __new__(cls, path: str, image: T_ARRAY, *args, **kwargs) -> None:
        """Alias of :meth:`ImageWriter.to`"""
        cls.to(path, image, *args, **kwargs)


# Databricks notebook source
# MAGIC %md The purpose of this notebook is train the model for the Clickstream Propensity solution accelerator.  You may find this notebook at https://github.com/databricks-industry-solutions/clickstream-analytics

# COMMAND ----------

# MAGIC %md ##Introduction
# MAGIC
# MAGIC With our metrics in place, we now can use those values to train a model.  Our model will predict the probability that a product engaged as part of a shopping session will be purchased within that session.  Such propensity calculations are typically handled as binary classification problems but ones where there's often a class imbalance created by the fact that most products interacted with are not ultimately purchased.  To overcome this issue, we will need to explore a variety of hyperparameter settings that will force our model to home in on those limited events associated with a purchase. That exercise, referred to as hyperparameter tuning, will require us to explore a large number of model variations.  We'll take advantage of the scalability of Databricks to speed up this process.

# COMMAND ----------

# DBTITLE 1,Get Config Info
# MAGIC %run "./0a_Intro & Config"

# COMMAND ----------

# DBTITLE 1,Import Required Libraries
import mlflow
from hyperopt import hp, fmin, tpe, SparkTrials, STATUS_OK, space_eval

from sklearn.model_selection import train_test_split
from sklearn.metrics import average_precision_score, balanced_accuracy_score, matthews_corrcoef

from xgboost import XGBClassifier

import pyspark.sql.functions as fn

# COMMAND ----------

# MAGIC %md ##Step 1: Retrieve Features & Labels
# MAGIC
# MAGIC Our first step is to assemble a set of features and labels. The metrics we calculated in our last notebook will serve as our features.  For the labels, we will flag each event record with a 0 or 1 indicating whether the product associated with that event eventually led to a purchase in the current shopping session. The first step in this process is to identify each product engaged in each shopping session and flag it based on whether it was purchased or not:

# COMMAND ----------

# DBTITLE 1,Identify Product Outcome in Shopping Sesssions
# identify whether a given product in a shopping session was purchased (1) or not (0)
cart_product_outcomes = (
  spark
    .table('electronics_events_silver')
    .groupBy('user_id','user_session','product_id')
      .agg(
        fn.max(fn.expr("CASE WHEN event_type='purchase' THEN 1 ELSE 0 END")).alias('purchased')
        )
  )

display(cart_product_outcomes.orderBy('user_id','product_id').limit(100))

# COMMAND ----------

# MAGIC %md Now we will assemble the feature records, one for each event in our historical dataset.  We will link each of these events to our various metrics, as those metrics were understood at the time of the event:

# COMMAND ----------

# DBTITLE 1,Assemble Features for Each Event Instance
# identify event instances to which we need to assign feature metrics
raw_features = (
  spark
    .table('electronics_events_silver')
    .select('user_id','user_session','product_id','event_time','event_type') # include event type for validation/troubleshooting purposes
    .withColumn('batch_time', fn.expr("DATE_TRUNC('day', event_time)")) # calculate batch time as would be available at this event time
  )

# identify metrics tables and how they will join with event data
metrics_map = { # table_name, key fields
  'electronics_cart_metrics__training':['user_id','user_session','event_time'],
  'electronics_cart_product_metrics__training':['user_id','user_session','product_id','event_time'],
  'electronics_user_metrics__training':['user_id','batch_time'],
  'electronics_product_metrics__training':['product_id','batch_time'],
  'electronics_user_product_metrics__training':['user_id','product_id','batch_time']
  }

# attach metrics to each event record
for table_name, key_fields in metrics_map.items():

  # retrieve metrics
  temp_features = spark.table(table_name)

  # identify how fields from metric table should be named in feature set
  prefix = table_name.split('__')[0] # prefix is everything prior to the __
  renamed_fields = [f"{c} as {prefix}__{c}" for c in temp_features.columns if c not in key_fields]

  # rename metric fields in preparation for join
  temp_features = temp_features.selectExpr(key_fields + renamed_fields)

  # join metrics to event instances
  raw_features = (
    raw_features
      .join(
        temp_features,
        on=key_fields,
        how='left'
        )
    )

# display feature data
display(raw_features.limit(100))

# COMMAND ----------

# MAGIC %md Now we will flag each event record for the eventual outcome of the associated product within the shopping session (cart), assigning a label to each feature instance:

# COMMAND ----------

# DBTITLE 1,Assemble Features & Labels
raw_features_and_labels = (
  raw_features
    .join(
      cart_product_outcomes,
      on=['user_id','user_session','product_id']
      )
  )

display(raw_features_and_labels.orderBy('user_id','user_session','event_time').limit(100))

# COMMAND ----------

# MAGIC %md In our raw features and labels dataset, we've kept a number of fields in place that are helpful for validation of the data. Having had a chance to review our data and now confident in the feature set, we can remove those fields.  We can also remove those records for the actual purchase events; just doesn't make sense to predict a purchase when a purchase occurs:
# MAGIC
# MAGIC **NOTE** We are keeping *user_id* and *user_session* so that we may split our data at a shopping cart level in the next step. Those fields will be removed once we have performed the split.

# COMMAND ----------

# DBTITLE 1,Finalize Features & Labels Set
features_and_labels = (
  raw_features_and_labels
    .filter("event_type != 'purchases'") # remove purchase events
    .drop('event_type','product_id','event_time','batch_time') # drop unnecessary fields
  )

# COMMAND ----------

# MAGIC %md ##Step 2: Define & Tune Model
# MAGIC
# MAGIC With our features and labels assembled, need to consider how best to split our data.  The typical pattern is to treat each event instance as independent of the others and perform a random split on that set.  If we did that, separate events from the same shopping cart might land in both our training and testing sets creating the potential for contamination where knowledge about a session in training influences the prediction we make during testing.  To avoid this, we may split on the individual shopping carts and then train on the events associated with the training carts and test on the events associated with the testing carts:

# COMMAND ----------

# DBTITLE 1,Get Unique Carts
carts = (
  features_and_labels
    .select('user_id','user_session')
    .distinct()
  )

# COMMAND ----------

# DBTITLE 1,Split on Carts
holdout_ratio = 0.1
train_test_ratio = 1 - holdout_ratio

# split carts
train_test, validate = carts.randomSplit([train_test_ratio, holdout_ratio])
train, test = train_test.randomSplit([1-(holdout_ratio/train_test_ratio), holdout_ratio/train_test_ratio])

# present split set counts
print(f"Training:\t{train.count()}")
print(f"Testing:\t{test.count()}")
print(f"Validate:\t{validate.count()}")

# COMMAND ----------

# DBTITLE 1,Associate Events with Split Carts
train_test = train_test.join(features_and_labels, on=['user_id','user_session']).drop('user_id','user_session')
train = train.join(features_and_labels, on=['user_id','user_session']).drop('user_id','user_session')
test = test.join(features_and_labels, on=['user_id','user_session']).drop('user_id','user_session')
validate = validate.join(features_and_labels, on=['user_id','user_session']).drop('user_id','user_session')

# present split set counts
print(f"Training:\t{train.count()}")
print(f"Testing:\t{test.count()}")
print(f"Validate:\t{validate.count()}")

# COMMAND ----------

# MAGIC %md With our data split, we're just about ready to perform hyperparameter tuning.  One of the key things we need to consider in doing such an exercise when our approach involves a binary classification is whether or not our data includes a class imbalance:

# COMMAND ----------

# DBTITLE 1,Examine Ratio of Positive and Negative Classes at Cart Level
display(
  features_and_labels
    .select('user_id','user_session','purchased') # get distinct sessions and session labels
    .distinct()
    .withColumn('total',fn.expr("COUNT(*) OVER()"))
    .groupby('purchased')
      .agg(
        fn.count('*').alias('instances'),
        fn.first('total').alias('total')
        )
    .withColumn('ratio',fn.expr("instances/total"))
    .drop('total')
  )

# COMMAND ----------

# MAGIC %md We can see from these results that only about 5% of carts result in a purchase.  It would be easy for us to state that no carts result in a purchase and we've have a model that's 95% accurate but clearly not useful.  So, we'll want to consider weighing the positive class instances a bit higher during our tuning exercise in order to force our model to give more consideration to positive class predictions.
# MAGIC
# MAGIC With this in mind, we can turn our attention to our model.  We'll make use of the [XGBoostClassifier](https://xgboost.readthedocs.io/en/stable/python/python_api.html#xgboost.XGBClassifier) given its frequent use in these kind of exercises.
# MAGIC
# MAGIC One of the challenges in using the XGBoostClassifier is the large number of hyperparameters with which model training is configured.  We'll define a search space for a subset of these parameters and then use [Hyperopt](http://hyperopt.github.io/hyperopt/) to intelligently explore the values in the identified ranges to see which combination of hyperparameter values give us our best model results. More details about defining Hyperopt search spaces can be found [here](http://hyperopt.github.io/hyperopt/getting-started/search_spaces/):

# COMMAND ----------

# DBTITLE 1,Define Hyperparameter Search Space
search_space = {
    'max_depth' : hp.quniform('max_depth', 1, 20, 1)
    ,'learning_rate' : hp.uniform('learning_rate', 0.01, 0.40)
    ,'scale_pos_weight' : hp.uniform('scale_pos_weight', 1.0, 100)
    }

# COMMAND ----------

# MAGIC %md To search this space, we need to define a function.  Hyperopt will pass a set of candidate hyperparameter values to this function which will return an evaluation metric with which the Hyperopt engine will evaluate parameter performance. That metric is returned as a loss value which Hyperopt seeks to minimize.  We will be using the average precision metric for our evaluation metric, which places greater emphasis on positive class predictions.  This metric *improves* as its value increases.  For that reason, we'll return a negative average precision score from the function so that Hyperopt drives the hyperparameters in the right direction:
# MAGIC
# MAGIC **NOTE** We've included some additional metrics in our evaluation as these are frequently used in evaluating binary classifiers with class imbalances.

# COMMAND ----------

# DBTITLE 1,Define Function for Evaluating Hyperparameter values
def evaluate_model(hyperopt_params):

  # accesss replicated input data
  train_input = train_pd_broadcast.value
  test_input = test_pd_broadcast.value

  X_train = train_input.drop('purchased', axis=1)
  y_train = train_input['purchased']
  X_test = test_input.drop('purchased', axis=1)
  y_test = test_input['purchased']

  # configure model parameters
  params = hyperopt_params
  if 'max_depth' in params: params['max_depth']=int(params['max_depth'])   # hyperopt supplies values as float but must be int
  if 'min_child_weight' in params: params['min_child_weight']=int(params['min_child_weight']) # hyperopt supplies values as float but must be int
  if 'max_delta_step' in params: params['max_delta_step']=int(params['max_delta_step']) # hyperopt supplies values as float but must be int
  # all other hyperparameters are taken as given by hyperopt

  # instantiate model with parameters
  model = XGBClassifier(**params)

  # train
  model.fit(X_train, y_train)

  # predict
  y_pred = model.predict(X_test)
  y_prob = model.predict_proba(X_test)

  # eval metrics
  model_ap = average_precision_score(y_test, y_prob[:,1])
  model_ba = balanced_accuracy_score(y_test, y_pred)
  model_mc = matthews_corrcoef(y_test, y_pred)

  # log metrics with mlflow run
  mlflow.log_metrics({
    'avg precision':model_ap,
    'balanced_accuracy':model_ba,
    'matthews corrcoef':model_mc
    })

  # invert key metric for hyperopt
  loss = -1 * model_ap

  # return results
  return {'loss': loss, 'status': STATUS_OK}

# COMMAND ----------

# MAGIC %md Within our function, we are tapping into variables named *train_pd_broadcast* and *test_pd_broadcast*.  These represent replicated copies of our *train* and *test* Spark dataframes converted to pandas dataframes.  (Note that if your data is too large to be moved to a pandas dataframe, you might consider taking a random sample your datasets.) By replicating these across each worker in our cluster, we enable each call to our function, multiple of which are executed in parallel across the Databricks cluster by Hyperopt, to access a local copy of the dataset.  But to create these copies of our data, we first need to broadcast the datasets as follows:

# COMMAND ----------

# DBTITLE 1,Broadcast Training & Testing Data Sets
train_pd_broadcast = sc.broadcast(train.toPandas())
test_pd_broadcast = sc.broadcast(test.toPandas())

# COMMAND ----------

# MAGIC %md With everything in place, we can now run our hyperparameter tuning jobs and identify our optimal parameters as follows.  Please note, we have limited our evaluations to relatively small number but may find even better results by elevating that value.  Using Hyperopt in combination with the scaled-out Databricks cluster, we are in control of how much time we want to spend seeking optimal hyperparameter values:

# COMMAND ----------

# DBTITLE 1,Explore Search Space
# set mlflow experiment so that this block works in a job
username = dbutils.notebook.entry_point.getDbutils().notebook().getContext().userName().get()
mlflow.set_experiment('/Users/{}/clickstream'.format(username))

# perform evaluation
with mlflow.start_run(run_name='tuning'):

  argmin = fmin(
    fn=evaluate_model,
    space=search_space,
    algo=tpe.suggest,  # algorithm controlling how hyperopt navigates the search space
    max_evals=50,
    trials=SparkTrials(parallelism=5),  # 50 evals with 5 evals at a time = 10 learning cycles
    verbose=True
    )

# separate hyperopt output from our results
print('\n')

# capture optimized hyperparameters
hyperopt_params = space_eval(search_space, argmin)
hyperopt_params

# COMMAND ----------

# MAGIC %md ##Step 4: Train & Persist Final Model
# MAGIC
# MAGIC With our optimal hyperparameter values identified, we can train our final model using all our training and testing data as the inputs and the validation set which was withheld from the tuning exercise for a final model evaluation:

# COMMAND ----------

# DBTITLE 1,Train Final Model & Validate
with mlflow.start_run(run_name='trained'):

  # get data train & validate final model with
  train_test_pd = train_test.toPandas()
  validate_pd = validate.toPandas()

  # separate X & y
  X_train_test = train_test_pd.drop('purchased', axis=1)
  y_train_test = train_test_pd['purchased']
  X_validate = validate_pd.drop('purchased', axis=1)
  y_validate = validate_pd['purchased']

  # configure model parameters
  params = hyperopt_params
  if 'max_depth' in params: params['max_depth']=int(params['max_depth'])   # hyperopt supplies values as float but must be int
  if 'min_child_weight' in params: params['min_child_weight']=int(params['min_child_weight']) # hyperopt supplies values as float but must be int
  if 'max_delta_step' in params: params['max_delta_step']=int(params['max_delta_step']) # hyperopt supplies values as float but must be int

  # train
  model = XGBClassifier(**params)
  model.fit(X_train_test, y_train_test)

  # predict
  y_pred = model.predict(X_train_test)
  y_prob = model.predict_proba(X_train_test)

  # eval metrics
  model_ap = average_precision_score(y_train_test, y_prob[:,1])
  model_ba = balanced_accuracy_score(y_train_test, y_pred)
  model_mc = matthews_corrcoef(y_train_test, y_pred)

  # record metrics with mlflow run
  mlflow.log_metrics({
    'avg precision':model_ap,
    'balanced_accuracy':model_ba,
    'matthews corrcoef':model_mc
    })

  # persist wrapped model
  mlflow.sklearn.log_model(
    artifact_path='model',
    sk_model=model,
    pip_requirements=['xgboost'],
    registered_model_name=config['model name'],
    pyfunc_predict_fn='predict_proba'
    )

  print(model_ap)

# COMMAND ----------

# MAGIC %md Please note two things about how we are logging our model.  First, we log it under the *sklearn* API.  The XGBoosClassifier model supports an sklearn API so that while there are both *xgboost* and *pyfunc* model flavors available to us in mlflow, we are able to log this model as an sklearn variant.
# MAGIC
# MAGIC Why do we do that?  Well, that's the second thing we need to note.  If you look at the *log_model* method call, you will see there is an *pyfunc_predict_fn* argument that's only supported with the *sklearn* mlflow flavor.  By setting this argument to *predict_proba* instead of allowing it to default to *predict* we are telling mlflow to use this alternative method to generate predictions when we use mlflow to deploy our model to a user-defined function or under an REST API.  This function returns the underlying probabilities associated with our class predictions which are far more useful in this scenario than simple 0s and 1s.
# MAGIC
# MAGIC Our model is now registered with the [MLFlow registry](https://mlflow.org/docs/latest/model-registry.html).  The registry supports integration with various CI/CD processes, but to facilitate our demonstration, we'll simply move our model to production status programmatically:

# COMMAND ----------

# DBTITLE 1,Move Model to Production Status
# connect to mlflow
client = mlflow.tracking.MlflowClient()

# identify model version in registry
model_version = client.search_model_versions(f"name='{config['model name']}'")[0].version

# move model version to production
client.transition_model_version_stage(
  name=config['model name'],
  version=model_version,
  stage='production'
  )

# COMMAND ----------

# MAGIC %md ##Step 4: Examine Predictions
# MAGIC
# MAGIC Before moving on, it would be helpful to examine our model's outputs to better understand the predictions they return.  To do this, we'll retrieve our model and deploy it as a udf:

# COMMAND ----------

# DBTITLE 1,Deploy Model as UDF
predict_udf = mlflow.sklearn.pyfunc.spark_udf(
  spark,
  f"models:/{config['model name']}/production"
  )

# COMMAND ----------

# MAGIC %md Before using our UDF to generate predictions, its important to understand what it will return.  The *predict_proba* method we configured it to use returns a 2-dimensional array.  The first value in the array represents the probability of class 0 (not purchased) and the second value represents the probability of class 1 (purchased).  However, the UDF only returns the first of these values which means we need to subtract the returned value from 1.0 in order to get the prediction we actually want:

# COMMAND ----------

# DBTITLE 1,Examine UDF Return Type
predict_udf.returnType

# COMMAND ----------

# MAGIC %md With that understood, we can now generate purchase predictions as follows:

# COMMAND ----------

# DBTITLE 1,Generate Predictions for a Sample Session
# get predictions for sample shopping session
sample = (
  raw_features_and_labels
    .filter(fn.expr("user_id=1515915625353900095 AND user_session='12dadbda-acb5-4f26-a5e8-f71814190c04'"))
    .withColumn('prediction', 1 - predict_udf(fn.struct([c for c in train.columns if c not in ['purchased']]))) # identify which columns to pass to the function for prediction
    .select('user_id','user_session','event_time','event_type','product_id','purchased','prediction')
    .orderBy('event_time')
  )

display(sample)

# COMMAND ----------

# MAGIC %md While every shopping experience is different, there are some interesting patterns here that are likely applicable to a large number of shopping sessions.  First, when our user first engages our products, we have a small bump in our belief they will complete the transaction.  But as the session continues on over an extended period of time, we settle into a general, low-level expectation they will make the purchase.  As we get to the end of this session, we see that we start picking up signal slowly boosts our expectation they will buy a product until we get right up to the end when our prediction moves close to certain.
# MAGIC
# MAGIC In this shopping session, there are some views that take place after the actual purchase which are flagged as certain of the purchase.  As was mentioned in the earlier notebook, this messiness in the data is not perfectly understood so these predictions are interesting but not too telling from a business execution standpoint.

# COMMAND ----------

# MAGIC %md © 2023 Databricks, Inc. All rights reserved. The source in this notebook is provided subject to the Databricks License.


from django.db import models
from common.models import CommonModel

# Create your models here.
class Review(CommonModel):
    """review from a User to a Room Experience"""

    user = models.ForeignKey(
        "users.User",
        on_delete=models.CASCADE,  # 유저 삭제 -> 리뷰도 삭제
        related_name="reviews",
    )
    room = models.ForeignKey(
        "rooms.Room",
        null=True,
        blank=True,
        on_delete=models.CASCADE,
        related_name="reviews",
    )
    experience = models.ForeignKey(
        "experiences.Experience",
        blank=True,
        null=True,
        on_delete=models.CASCADE,
        related_name="reviews",
    )
    payload = models.TextField()
    rating = models.PositiveIntegerField()

    def __str__(self) -> str:
        return f"{self.user} 님의 별점 : {self.rating}"


import mock

from os_brick import exception
from os_brick.initiator.connectors import sheepdog
from os_brick.initiator import linuxsheepdog
from os_brick.tests.initiator import test_connector

class SheepdogConnectorTestCase(test_connector.ConnectorTestCase):

    def setUp(self):
        super(SheepdogConnectorTestCase, self).setUp()

        self.hosts = ['fake_hosts']
        self.ports = ['fake_ports']
        self.volume = 'fake_volume'

        self.connection_properties = {
            'hosts': self.hosts,
            'name': self.volume,
            'ports': self.ports,
        }

    def test_get_connector_properties(self):
        props = sheepdog.SheepdogConnector.get_connector_properties(
            'sudo', multipath=True, enforce_multipath=True)

        expected_props = {}
        self.assertEqual(expected_props, props)

    def test_get_search_path(self):
        sd_connector = sheepdog.SheepdogConnector(None)
        path = sd_connector.get_search_path()
        self.assertIsNone(path)

    def test_get_volume_paths(self):
        sd_connector = sheepdog.SheepdogConnector(None)
        expected = []
        actual = sd_connector.get_volume_paths(self.connection_properties)
        self.assertEqual(expected, actual)

    def test_connect_volume(self):
        """Test the connect volume case."""
        sd_connector = sheepdog.SheepdogConnector(None)
        device_info = sd_connector.connect_volume(self.connection_properties)

        # Ensure expected object is returned correctly
        self.assertIsInstance(device_info['path'],
                              linuxsheepdog.SheepdogVolumeIOWrapper)

    @mock.patch.object(linuxsheepdog.SheepdogVolumeIOWrapper, 'close')
    def test_disconnect_volume(self, volume_close):
        """Test the disconnect volume case."""
        sd_connector = sheepdog.SheepdogConnector(None)
        device_info = sd_connector.connect_volume(self.connection_properties)
        sd_connector.disconnect_volume(self.connection_properties, device_info)

        self.assertEqual(1, volume_close.call_count)

    def test_disconnect_volume_with_invalid_handle(self):
        """Test the disconnect volume case with invalid handle."""
        sd_connector = sheepdog.SheepdogConnector(None)
        device_info = {'path': 'fake_handle'}
        self.assertRaises(exception.InvalidIOHandleObject,
                          sd_connector.disconnect_volume,
                          self.connection_properties,
                          device_info)

    def test_extend_volume(self):
        sd_connector = sheepdog.SheepdogConnector(None)
        self.assertRaises(NotImplementedError,
                          sd_connector.extend_volume,
                          self.connection_properties)


from asyncio import Lock, ensure_future, gather, sleep
from contextlib import asynccontextmanager, contextmanager
from random import random
from typing import AsyncIterable, Dict, Optional, Set

from aiogoogle import HTTPError

from ..aiogoogle_service_account import AiogoogleServiceAccount
from ..config import ConfigSchema
from ..helpers import dict_drop_blanks
from ..models import GoogleGroup, GoogleGroupMember

class GoogleAPIIntegration(object):
    def __init__(
        self, config: ConfigSchema, aiogoogle: AiogoogleServiceAccount = AiogoogleServiceAccount
    ) -> None:
        self._client_creds = config.google._asdict()
        self._aiogoogle: AiogoogleServiceAccount = aiogoogle
        self._domain = config.domain
        self._lock = Lock()

        self._admin_api = None
        self._groups_api = None

        # Etag for the users.list endpoint
        # Optimises performance when user aliases aren't updated
        self._users_etag = None

        # Allow up to 5 requests every 335ms
        # Sums to ~15 per second, 1500 per 100 seconds
        # Rate limit on google is 1500 per 100 seconds
        # There is no race condition on these ints, because
        # asyncio is not threaded or multi process
        self._request_rate = 5
        self._request_interval = 0.335
        self._request_credits = 0
        self._spent_credits = 0

        # Track exponential backoff
        self._pause_interval = 1
        self._pause_max = 30
        self._run_regulator = False

    async def _pressure_regulator(self) -> None:
        self._request_credits = 0
        self._spent_credits = 0
        while self._run_regulator:
            await sleep(self._request_interval)
            self._request_credits += self._request_rate

    @contextmanager
    def pressure_regulator(self) -> any:
        self._run_regulator = True
        ensure_future(self._pressure_regulator())
        try:
            yield
        finally:
            self._run_regulator = False
            print("Google Groups Integration ran", self._spent_credits, "requests")

    async def _spend_credits(self, amount: int = 1) -> None:
        while self._request_credits < amount:
            # Add some variance so that if there's a bunch of requests
            # they don't run at the same time
            await sleep(self._pause_interval + (random() * 3))
            # Check now if the credits have increased enough. If not, wait longer
            if self._request_credits < amount:
                self._pause_interval = min(self._pause_interval ** 2, self._pause_max)
        self._request_credits -= amount
        self._spent_credits += amount

    @asynccontextmanager
    async def get_client(self, client: AiogoogleServiceAccount = None) -> AiogoogleServiceAccount:
        if client:
            yield client
        else:
            # Hacky check to inform console that the API is locked
            # Useful when there are bad crashes
            if self._run_regulator:
                print("Waiting for Google API lock")
            async with self._lock:
                with self.pressure_regulator():
                    async with self._aiogoogle(client_creds=self._client_creds) as aiog:

                        # Cache the API discovery
                        if self._admin_api is None or self._groups_api is None:
                            await self._spend_credits(4)
                            self._admin_api = await aiog.discover("admin", "directory_v1")
                            self._groups_api = await aiog.discover("groupssettings", "v1")

                        yield aiog

    async def _load_members(self, aiog: AiogoogleServiceAccount, group: GoogleGroup) -> None:
        await self._spend_credits(1)
        response = await aiog.as_user(
            self._admin_api.members.list(groupKey=group.group_id), full_res=True
        )

        async for page in response:
            await self._spend_credits(1)

            # If it has no members, the key doesn't even exist :scream:
            for raw_member in page.get("members", []):
                group.add_member(GoogleGroupMember.from_api(raw_member))

    async def _get_protected_status(
        self, aiog: AiogoogleServiceAccount, group: GoogleGroup
    ) -> None:
        await self._spend_credits(1)
        try:
            response = await aiog.as_user(
                self._groups_api.groups.get(groupUniqueId=group.email, alt="json")
            )
            group.protected = response["whoCanJoin"] == "INVITED_CAN_JOIN"
        except HTTPError as err:
            # Rate limiting
            if err.res.status_code == 403:
                self._request_credits = 0
                sleep(self._pause_interval)
                return self._get_protected_status(aiog, group)
            else:
                print(f"Error getting group '{group.email}' protected status: ", err)

    async def load_groups(
        self, etags: Dict[str, str] = None, client: AiogoogleServiceAccount = None
    ) -> AsyncIterable[GoogleGroup]:
        etags = etags or {}

        async with self.get_client(client) as aiog:

            # Load all groups in the domain
            await self._spend_credits(1)
            response = await aiog.as_user(
                self._admin_api.groups.list(domain=self._domain), full_res=True
            )

            # Parse them into GoogleGroup objects
            async for page in response:
                await self._spend_credits(1)

                for raw_group in page["groups"]:
                    group = GoogleGroup.from_api(raw_group)

                    # Check the etag
                    if group.etag != etags.get(group.group_id):
                        # Load members and settings
                        # No point doing this in one big gather because Google <3 rate limits
                        await gather(
                            self._load_members(aiog, group), self._get_protected_status(aiog, group)
                        )

                    yield group

    async def load_user_emails(
        self, client: AiogoogleServiceAccount = None
    ) -> AsyncIterable[Set[str]]:

        async with self.get_client(client) as aiog:

            # Load all users in the domain
            # The emails field is all that is needed.
            # It includes primaryEmail and aliases
            await self._spend_credits(1)
            response = await aiog.as_user(
                self._admin_api.users.list(
                    domain=self._domain,
                    fields="etag,users(emails(address))",
                    projection="full",
                    maxResults=200,
                ),
                full_res=True,
            )

            new_etag: str = ""
            async for page in response:
                await self._spend_credits(1)
                new_etag = new_etag or page["etag"]

                if page["etag"] == self._users_etag:
                    return

                for raw_user in page["users"]:
                    yield {
                        email["address"]
                        for email in raw_user["emails"]
                        if "test-google" not in email["address"]
                    }

            self._users_etag = new_etag

    async def add_group_member(
        self,
        group: GoogleGroup,
        email: str,
        role: str = "MEMBER",
        client: AiogoogleServiceAccount = None,
    ) -> Optional[GoogleGroupMember]:
        async with self.get_client(client) as aiog:

            # Add the member
            try:
                await self._spend_credits(1)
                response = await aiog.as_user(
                    self._admin_api.members.insert(
                        groupKey=group.group_id,
                        json={
                            "email": email,
                            "role": role,
                            "kind": "admin#directory#member",
                            "status": "ACTIVE",
                        },
                    )
                )
            except HTTPError as err:
                if (
                    "member already exists"
                    in err.res.content.get("error", {}).get("message", "").lower()
                ):
                    return
                raise err

            if not response:
                raise ValueError("Member insert request failed: Empty response")

            return GoogleGroupMember.from_api(response)

    async def remove_group_member(
        self, group: GoogleGroup, member: GoogleGroupMember, client: AiogoogleServiceAccount = None,
    ) -> Optional[str]:
        async with self.get_client(client) as aiog:

            # Remove the member
            # TODO idempotent
            await self._spend_credits(1)
            response = await aiog.as_user(
                self._admin_api.members.delete(groupKey=group.group_id, memberKey=member.member_id)
            )

            # Returns None..on success.
            if response and response.status_code >= 299:
                raise ValueError(response.reason or f"Status code {response.status_code}")

    async def add_group(
        self,
        email: str,
        name: str = "",
        description: str = "",
        protect: bool = True,
        client: AiogoogleServiceAccount = None,
    ) -> GoogleGroup:
        async with self.get_client(client) as aiog:

            # Add the group
            await self._spend_credits(1)
            response = await aiog.as_user(
                self._admin_api.groups.insert(
                    json=dict_drop_blanks(
                        {"email": email, "name": name, "description": description}
                    ),
                )
            )

            if not response:
                raise ValueError("Group creation request failed: Empty response")

            group = GoogleGroup.from_api(response)
            who_can_join = "INVITED_CAN_JOIN" if protect else "ALL_IN_DOMAIN_CAN_JOIN"

            await self._spend_credits(1)
            response = await aiog.as_user(
                self._groups_api.groups.patch(
                    groupUniqueId=email, json={"whoCanJoin": who_can_join}
                )
            )

            if not response:
                raise ValueError("Group creation request failed: Failed to set protected status")

            return group


#! /usr/bin/python2
from interruptingcow import timeout

def get_exp(number):
    exp = 1
    while number % 10 == 0:
        exp *= 10
        number //= 10
    return exp

def get_start_value(number):
    nine_amount = number // 9
    i = int(str(number % 9) + ('9' * nine_amount))
    i *= get_exp(number)
    return i

class Number:
    def __init__(self, number, n):
        self.number = number
        # getting endian_exp
        self.endian_exp = 10  # self.calc_endian_exp()
        # init little_exp
        self.little_exp = 1
        # save n
        self.n = n
        self.n_exp = 1
        while (n % 10) == 0:
            n //= 10
            self.n_exp *= 10

    def get_little_digit(self):
        return (self.number // self.little_exp) % 10

    def get_endian_digit(self):
        return (self.number // self.endian_exp) % 10

    def calc_endian_exp(self):
        return int(10 ** (len(str(self.number)) - 1))

    def find_littles_free_exp(self):
        exp = self.endian_exp * 10
        num = self.number // exp
        while num % 10 == 9:
            num //= 10
            exp *= 10
        return exp

    def find_biggest_unfree_exp(self):
        exp = self.endian_exp
        while (self.number // (exp * 10)) % 10 == 9:
            exp *= 10
        return exp

    def get_digit(self, exp):
        return (self.number // exp) % 10

    def carry_one(self):
        # carrying
        littles_exp = self.find_littles_free_exp()
        unfree_exp = littles_exp // 10
        self.number += littles_exp
        # putting remained digits
        remain = self.get_digit(unfree_exp) - 1
        self.number -= unfree_exp * self.get_digit(unfree_exp)
        e = 1
        while remain:
            d = (self.number // e) % 10
            if d < 9:
                diff = min(9 - d, remain)
                self.number += e * diff
                remain -= diff
            e *= 10

    def sophisticated_increase(self):
        # finding minimal digit position that can be increased
        exp = 1
        s = 0
        while (self.number // exp) % 10 == 9 or s == 0:
            s += (self.number // exp) % 10
            exp *= 10
        # building minimal number
        n = (self.number // exp) * exp
        n += exp
        s -= 1
        e = self.n_exp
        while s >= 9:
            n += e * 9
            e *= 10
            s -= 9
        n += e * s
        # updating number
        self.number = n

    def increment(self):
        if self.get_endian_digit() < 9 and self.get_little_digit() > 0:
            self.number += self.endian_exp - self.little_exp
            return self.number
        elif self.get_little_digit() == 0:
            self.sophisticated_increase()
        else:
            self.carry_one()
        return self.get()

    def get(self):
        return self.number

def find_answer(n):
    if n < 10:
        return n
    start_value = get_start_value(n)
    if start_value % n == 0:
        return start_value
    i = Number(start_value, n)
    while i.increment() % n != 0:
        pass
    return i.get()

db = {}
# print find_answer(int(input()))
r = [i for i in range(1, 1000)]
r.remove(165)
r.remove(185)
r.remove(275)
r.remove(297)
r.remove(308)
with open('db.txt', 'r') as f:
    # getting and removing already calculated
    for l in f.readlines():
        j = int(l.split(':')[0])
        r.remove(j)
total_skipped = 0
with open('db.txt', 'a') as f:
    for i in r:
        try:
            with timeout(45, exception=RuntimeError):
                print('Saving for ', i)
                db[i] = find_answer(i)
                f.write(str(i) + ':' + str(db[i]) + '\n')
                if i % 25 == 0:
                    f.flush()
        except RuntimeError:
            print('Timeout! Skipping')
            total_skipped += 1

print('Total skipped: ', total_skipped)


from tweepy import API
from tweepy import Cursor
from tweepy import OAuthHandler
import json
import credentials
import re
import matplotlib.pyplot as plt
import numpy as np

# Imports the Google Cloud client library
from google.cloud import language
from google.cloud.language import enums
from google.cloud.language import types

# Instantiates a client
class Visual:

  def plot_graph(self, filepath):
    with open(filepath, 'r') as file_obj:
      data = json.load(file_obj)
      file_obj.close()
  score = np.zeros(len(data),dtype=int)

  for i,value in enumerate(data.values()):
    score[i] = value*10
  x = score
  # the histogram of the data
  n, bins, patches = plt.hist(x, 21, density=True, facecolor='g', alpha=0.99, histtype= 'stepfilled')
  plt.xlabel('sentiment')
  plt.ylabel('Probability')
  plt.title('Sentiment of Hong Kong Protest')
  plt.xlim(-10, 10)
  plt.ylim(0, 0.4)
  plt.grid(True)
  plt.show()

class Analysis:

  def __init__(self):
    self.client = language.LanguageServiceClient()

  def tweets_analysis(self, filepath, filepath_res):
    with open(filepath, 'r') as load_f:
      tweets = json.load(load_f)
      print(tweets[0]['full_text'])
    # The text to analyze
      score = {}
      magnitude = {}
      for i in range(len(tweets)):
        document = types.Document(
        content=tweets[i]['full_text'],
        type=enums.Document.Type.PLAIN_TEXT)
        # Detects the sentiment of the text
        try:
          sentiment = self.client.analyze_sentiment(document=document).document_sentiment
        except BaseException:
          continue

        score[i] = sentiment.score
        magnitude[i] = sentiment.magnitude
        print('\n','\n')
        print('Text: {}'.format(tweets[i]['text']))
        print('Sentiment: {}, {}'.format(score[i], magnitude[i]))

      with open(filepath_res, 'w') as file_obj:
          json.dump(score, file_obj)
    print('Avarage Score:',sum(score.values())/len(score))
'''
class Analysis:
  # Instantiates a client
  def __init__(self):
    self.object = TweetsSearch()
    self.client = language.LanguageServiceClient()
# text analyze
  def tweets_analysis(self, query, time, tweets_number, subkey):
    texts = self.object.search_tweets_sorted(query, time, tweets_number, subkey)
    for text in texts:
      document = types.Document(content=text, type=enums.Document.Type.PLAIN_TEXT)
      # Detects the sentiment of the text
      sentiment = self.client.analyze_sentiment(document=document).document_sentiment
      print('Text: {}'.format(text))
      print('Sentiment: {}, {}'.format(sentiment.score, sentiment.magnitude))
'''

# this class is used for authentication.
class Authenticator:

  def get_authentication(self):
    auth = OAuthHandler(credentials.consumer_key, credentials.consumer_secret)
    auth.set_access_token(credentials.access_token, credentials.access_token_secret)
    return auth

# search by specific words
class TweetsSearch:

  def __init__(self):
    self.auth = Authenticator().get_authentication()
    self.user = API(self.auth)

  def search_tweets(self, query, time, filepath, tweets_number):
    tweets = []
    with open(filepath, 'w') as file:
      for tweet in Cursor(self.user.search, q=query, since=time).items(tweets_number):
        tmp_tweet = tweet._json
        tweets.append(tmp_tweet)
      tweet_json = json.dump(tweets,file,sort_keys = True,indent = 4)
      file.close()

  def search_tweets_sorted(self, query, time, filepath, tweets_number, subkey):
    tweets = []
    with open(filepath, 'w') as file:
      for tweet in Cursor(self.user.search, q=query, tweet_mode = 'extended', since=time).items(tweets_number):
        tmp_tweet = tweet._json
        tmp_tweet_new = tmp_tweet[subkey]
        # tmp_tweet_new = dict([(key, tmp_tweet[key]) for key in subkey])
        tweet_clean = re.sub(r"http\S+", "", tmp_tweet_new)
        tweets.append(tweet_clean)
      tweet_json = json.dump(tweets,file,sort_keys = True,indent = 4)
      file.close()

if __name__ == '__main__':

  example = TweetsSearch()
  sen_analysis = Analysis()
  visual = Visual()

  filepath = './test.json'
  filepath_res = './test_res.json'
  querys = ["#unitedAIRLINES"]
  time = "2019-09-20"
  # subkeys = ['created_at', 'id', 'full_text','retweeted']
  subkeys ='full_text'

  example.search_tweets_sorted(querys, time, 5, subkeys)
  sen_analysis.tweets_analysis(filepath, filepath_res)
  visual.plot_graph(filepath_res)


import math
import random

import pygame

pygame.font.init()
font = pygame.font.Font(None, 30)

FPS = 30

RED = 0xFF0000
BLUE = 0x0000FF
YELLOW = 0xFFC91F
GREEN = 0x00FF00
MAGENTA = 0xFF03B8
CYAN = 0x00FFCC
BLACK = (20, 20, 20)
WHITE = 0xFFFFFF
GREY = (120, 120, 120)
GAME_COLORS = [RED, BLUE, YELLOW, GREEN, MAGENTA, CYAN]
TANKGREEN = (100, 120, 70)

WIDTH = 800
HEIGHT = 600

pygame.init()
screen = pygame.display.set_mode((WIDTH, HEIGHT), pygame.SRCALPHA)

#Рисуем поверхность с танком
tanksurf = pygame.Surface((300, 300))
tanksurf.set_colorkey((0, 0, 0))
pygame.draw.circle(tanksurf, GREY, (50, 250), 40)
pygame.draw.circle(tanksurf, GREY, (250, 250), 40)
pygame.draw.rect(tanksurf, GREY, (50, 210, 200, 80))
pygame.draw.circle(tanksurf, BLACK, (50, 250), 30)
pygame.draw.circle(tanksurf, BLACK, (250, 250), 30)
pygame.draw.circle(tanksurf, BLACK, (100, 250), 30)
pygame.draw.circle(tanksurf, BLACK, (150, 250), 30)
pygame.draw.circle(tanksurf, BLACK, (200, 250), 30)
pygame.draw.polygon(tanksurf, TANKGREEN, ((0, 230), (60, 170), (60, 230)))
pygame.draw.polygon(tanksurf, TANKGREEN, ((300, 230), (240, 170), (240, 230)))
pygame.draw.rect(tanksurf, TANKGREEN, (60, 170, 180, 60))
pygame.draw.circle(tanksurf, TANKGREEN, (150, 170), 60)

#Размещаем поверхность с танком на заданной поверхности, в координатах x, y и с масштабом scale
def draw_tank(surface, x , y, scale=1, color = TANKGREEN):
    tanknewsurf = tanksurf
    pygame.draw.polygon(tanknewsurf, color, ((0, 230), (60, 170), (60, 230)))
    pygame.draw.polygon(tanknewsurf, color, ((300, 230), (240, 170), (240, 230)))
    pygame.draw.rect(tanknewsurf, color, (60, 170, 180, 60))
    pygame.draw.circle(tanknewsurf, color, (150, 170), 60)
    if scale != 1:
        tanksurf_tranformed = pygame.transform.scale(tanksurf, (round(300*scale), round(300*scale)))
        surface.blit(tanksurf_tranformed, (x, y))
    else:
        surface.blit(tanksurf, (x, y))

def help():
    clock_help = pygame.time.Clock()
    help_active = True
    help_text = []
    text = []

    help_text.append("Управление у первого игрока: ")
    help_text.append("   A, D - двигать танк; W, S - целиться;")
    help_text.append("   E - зажимать для стрельбы; R - перезарядка;")
    help_text.append("   Q - сменить тип патронов;")

    help_text.append("Управление у второго игрока: ")
    help_text.append("   стрелочки вправо и влево - двигать танк;")
    help_text.append("   стрелочки вверх и вниз - целиться;")
    help_text.append("   RSHIFT - зажимать для стрельбы;")
    help_text.append("   RCTRL - перезарядка;")
    help_text.append("   ENTER - сменить тип патронов;")

    help_text.append("Если патроны закончились - ")
    help_text.append("перезаряжаться нужно вручную.")

    help_text.append("Серый эллипс скидывает новые мишени,")
    help_text.append("только если их становится меньше 10.")

    for i in range(len(help_text)):
        text.append(font.render(help_text[i], True, BLACK))

    size_x = 500
    size_y = 400
    help_x = 150
    help_y = 125

    exitbutton = Button(screen, help_x + size_x/2 - 50, help_y + size_y - 60, 100, 50, "back", 1)

    while help_active:
        clock_help.tick(FPS)
        pygame.draw.rect(screen, WHITE, (0, 0, WIDTH, HEIGHT))

        exitbutton.draw()

        helpsurface2 = pygame.Surface((size_x, size_y))
        helpsurface2.set_colorkey((0, 0, 0))
        pygame.draw.rect(helpsurface2, GREY, (0, 0, size_x, size_y), 7)
        screen.blit(helpsurface2, (help_x + 3, help_y + 3))

        helpsurface1 = pygame.Surface((size_x, size_y))
        helpsurface1.set_colorkey((0, 0, 0))
        pygame.draw.rect(helpsurface1, BLACK, (0, 0, size_x, size_y), 7)
        screen.blit(helpsurface1, (help_x, help_y))

        y = help_y + 10
        for i in range(len(text)):
            screen.blit(text[i], (help_x + 10, y))
            y += text[i].get_height()

        pygame.display.update()

        for event in pygame.event.get():
            if event.type == pygame.QUIT:
                pygame.quit()
            if exitbutton.check(event):
                help_active = False

#Класс отвечающий за кнопки
class Button:
    def __init__(self, surface, x, y, width, height, buttontext, buttontype = 0):
        """
        :param surface: Поверхность для отрисовки кнопки
        :param x: Координата x левого верхнего угла кнопки
        :param y: Координата y левого верхнего угла кнопки
        :param width: Ширина кнопки
        :param height: Высота кнопки
        :param buttontext: Текст на кнопке
        """
        self.surface = surface
        self.x = x
        self.y = y
        self.width = width
        self.height = height
        self.buttontype = buttontype

        #Рисуем поверхность с черной кнопкой
        self.buttonsurf = pygame.Surface((self.width, self.height))
        self.buttonsurf.set_colorkey((0, 0, 0))
        pygame.draw.rect(self.buttonsurf, BLACK, (0, 0, self.width, self.height), 7)
        text = font.render(buttontext, True, (10, 10, 10))
        self.buttonsurf.blit(text, ((self.width - text.get_width())/2, (self.height - text.get_height())/2))

        #Рисуем поверхность с серой кнопкой, нужна для создания ээфекта обьема и анимации нажатия
        self.buttonshadow = pygame.Surface((self.width, self.height))
        self.buttonshadow.set_colorkey((0, 0, 0))
        pygame.draw.rect(self.buttonshadow, GREY, (0, 0, self.width, self.height), 7)
        text = font.render(buttontext, False, (150, 150, 150))
        self.buttonshadow.blit(text, ((self.width - text.get_width()) / 2, (self.height - text.get_height()) / 2))

        self.pressed = False
        self.nexttick = False

    def draw(self):
        # Рисуем кнопку по разному в зависимости от того нажата ли она
        if self.pressed and self.nexttick:
            self.pressed = False
            self.nexttick = False

        if not self.pressed:
            self.surface.blit(self.buttonshadow, (self.x + 3, self.y + 3))
            self.surface.blit(self.buttonsurf, (self.x, self.y))
        else:
            self.surface.blit(self.buttonshadow, (self.x + 3, self.y + 3))
            self.surface.blit(self.buttonsurf, (self.x + 2, self.y + 2))

    def check(self, event):
        #Проверка на нажатие кнопки
        if self.buttontype == 0:
            if event.type == pygame.MOUSEBUTTONDOWN:
                if self.x < event.pos[0] < self.x + self.width:
                    if self.y < event.pos[1] < self.y + self.height:
                        self.pressed = True
                        return True
            if event.type == pygame.MOUSEBUTTONUP and self.pressed == True:
                self.nexttick = True
            return False
        if self.buttontype == 1:
            if event.type == pygame.MOUSEBUTTONDOWN:
                if self.x < event.pos[0] < self.x + self.width:
                    if self.y < event.pos[1] < self.y + self.height:
                        self.pressed = True
            if event.type == pygame.MOUSEBUTTONUP and self.pressed == True:
                self.nexttick = True
                return True
            return False

#Класс для эффектов выхлопных газов от танка (создает летающие шарики серого цвета)
class Particle:
    def __init__(self, x, y, alpha, r, screen):
        """
        :param x: Координата x центра кружка
        :param y: Координата y центра кружка
        :param alpha: Начальная прозрачность кружка
        :param r: Начальный радиус кружука
        :param screen: Поверхность для отрисовки
        """
        self.x = x
        self.y = y
        self.r = r
        self.vy = 0
        self.vx = random.randint(-3, 3)
        self.velocity = 1
        self.alpha = alpha
        self.screen = screen
    def update(self):
        #Обновляем позицию на экране, скорость, прозрачность шарика
        self.vy += self.velocity
        self.y -= self.vy
        self.x += self.vx
        self.r += 2
        self.alpha -= 2
        self.partsurface = pygame.Surface((self.r*2, self.r*2))
        pygame.draw.circle(self.partsurface, GREY, (self.r, self.r), self.r)
        self.partsurface.set_colorkey((0,0,0))
        self.partsurface.set_alpha(self.alpha)
        self.screen.blit(self.partsurface, (self.x - self.r, self.y - self.r))

class Ball:
    def __init__(self, _screen, x=40, y=450):
        """
        :param _screen: Поверхность для отрисовки
        :param x: Начальная координата x
        :param y: Начальная координата y
        """
        self.screen = _screen
        self.x = x
        self.y = y
        self.r = 10
        self.vx = 0
        self.vy = 0
        self.color = random.choice(GAME_COLORS)
        self.live = 30
        self.alpha = 255

        self.max_c = 6
        self.c = 0

        self.parent = None

    def update(self):
        """Переместить мяч по прошествии единицы времени. И обработать столкновение со стенками.

        Метод описывает перемещение мяча за один кадр перерисовки. То есть, обновляет значения
        self.x и self.y с учетом скоростей self.vx и self.vy, силы гравитации, действующей на мяч,
        и стен по краям окна (размер окна 800х600).
        """
        self.x += self.vx
        self.y -= self.vy

        if self.x > WIDTH - self.r and self.vx > 0:
            self.vx *= -0.6
            self.vy *= 0.8
            self.x = WIDTH - self.r
            self.c += 1
        if self.y > HEIGHT - self.r and self.vy < 0:
            self.vy *= -0.6
            self.vx *= 0.8
            self.y = HEIGHT - self.r
            self.c += 1
        if self.x < self.r and self.vx < 0:
            self.vx *= -0.6
            self.vy *= 0.8
            self.x = self.r
            self.c += 1
        if self.c > self.max_c:
            if self.alpha > 0:
                self.alpha -= 4
            else:
                self.alpha = 0
        self.vy -= 1

    def draw(self):
        # Отрисовать шарик с учетом его прозрачности(После нескольких столкновений шарик плавно становится прозрачным и исчезает)
        self.surface1 = pygame.Surface((self.r*2, self.r*2))
        self.surface1.set_alpha(self.alpha)
        pygame.draw.circle(self.surface1, self.color, (self.r, self.r), self.r)
        self.surface1.set_colorkey((0, 0, 0))
        screen.blit(self.surface1, (self.x - self.r, self.y - self.r))

    def hittest(self, obj):
        """Функция проверяет сталкивалкивается ли данный обьект с целью, описываемой в обьекте obj.
        Args:
            obj: Обьект, с которым проверяется столкновение.
        Returns:
            Возвращает True в случае столкновения мяча и цели. В противном случае возвращает False.
            Если проверяем столкновение двух обьектов класса Ball, то меняем им скорости после столкновения.
        """
        dist = (self.x - obj.x) ** 2 + (self.y - obj.y) ** 2 - (self.r + obj.r) ** 2
        if dist > 0:
            return False
        else:
            if type(obj) == Ball:
                p1 = complex(self.x, self.y)
                p2 = complex(obj.x, obj.y)
                v1 = complex(self.vx, self.vy)
                v2 = complex(obj.vx, obj.vy)
                p12 = p1 - p2
                d = ((v1 - v2) / p12).real * p12
                self.vx = (v1 - d).real
                self.vy = (v1 - d).imag
                obj.vx = (v2 + d).real
                obj.vy = (v2 + d).imag
            else:
                self.alpha = 0
            return True

#Класс отвечающий за пушку и ее перемещение
class Gun:
    def __init__(self, screen, controltype = 1):
        self.screen = screen
        self.f2_power = 10
        self.f2_on = 0
        self.an = 1
        self.color = GREY

        self.timer = 0

        self.x = 100
        self.y = 540

        self.mousex = 100
        self.mousey = 100

        self.vx = 0
        self.maxspeed = 3
        self.velocity = 0

        self.moving_right = False
        self.moving_left = False

        self.reloadtime = self.defreloadtime = 90
        self.bulletsleft = self.defbulletsleft = 8

        self.fuel = 3000
        self.fuellinecolor = GREEN

        self.controltype = controltype
        self.anup = False
        self.andown = False

        self.k_move_forward = pygame.K_d
        self.k_move_backward = pygame.K_a
        self.k_targeting_up = pygame.K_w
        self.k_targeting_down = pygame.K_s
        self.k_fire = pygame.K_e
        self.k_reload = pygame.K_r
        self.changebullettype = pygame.K_q

        self.statx = 50
        self.staty = 50

        self.name = "Tank"
        self.color = TANKGREEN

        self.score = 0

        self.reloading = False

        self.bullettype = 0

    def setgunstats(self, k_move_forward, k_move_backward, k_targeting_up, k_targeting_down, k_fire, changebullettype, k_reload, statx, staty):
        self.k_move_forward = k_move_forward
        self.k_move_backward = k_move_backward
        self.k_targeting_up = k_targeting_up
        self.k_targeting_down = k_targeting_down
        self.k_fire = k_fire
        self.changebullettype = changebullettype
        self.k_reload = k_reload
        self.statx = statx
        self.staty = staty

    def fire2_start(self):
        #Начать увеличивать мощность пушки (зажать ЛКМ)
        if self.bulletsleft > 0:
            self.f2_on = 1

    def fire2_end(self, bullettype):
        """Выстрел мячом.

        Происходит при отпускании кнопки мыши.
        Начальные значения компонент скорости мяча vx и vy зависят от положения мыши.
        """
        global balls, bullet
        if self.bulletsleft > 0 and bullettype == 0:
            bullet += 1
            new_ball = Ball(self.screen, self.x + self.f2_power * math.cos(self.an), self.y + self.f2_power * math.sin(-self.an))
            new_ball.r += 5
            new_ball.vx = self.f2_power * math.cos(self.an)
            new_ball.vy = - self.f2_power * math.sin(-self.an)
            new_ball.parent = self
            balls.append(new_ball)
            self.f2_on = 0
            self.f2_power = 10
            self.bulletsleft -= 1
        if self.bulletsleft > 0 and bullettype == 1:
            bullet += 3
            for i in range(3):
                new_ball = Ball(self.screen, self.x + math.cos(self.an + (i-1)*0.3) * 30, self.y + math.sin(-self.an - (i-1)*0.3) * 30)
                new_ball.vx = self.f2_power * math.cos(self.an + (i-1)*0.3) / 1.3
                new_ball.vy = - self.f2_power * math.sin(-self.an - (i-1)*0.3) / 1.3
                new_ball.parent = self
                balls.append(new_ball)
            self.f2_on = 0
            self.f2_power = 10
            self.bulletsleft -= 1

    def targetting(self, event):
        """Прицеливание. Зависит от положения мыши."""
        if self.controltype == 0:
            if event.type == pygame.MOUSEMOTION:
                self.mousex = event.pos[0]
                self.mousey = event.pos[1]
            self.an = - math.atan2((self.mousey-self.y), (self.mousex-self.x))
            if self.f2_on:
                self.color = RED
            else:
                self.color = GREY

        if self.controltype == 1:
            if event.type == pygame.KEYDOWN:
                if event.key == self.k_targeting_up:
                    self.anup = True
                if event.key == self.k_targeting_down:
                    self.andown = True
            if event.type == pygame.KEYUP:
                if event.key == self.k_targeting_up:
                    self.anup = False
                if event.key == self.k_targeting_down:
                    self.andown = False

    def drawstats(self):
        text = font.render(self.name, True, (0, 0, 0))
        self.screen.blit(text, (self.statx, self.staty - 20))

        if self.reloading:
            text = font.render("Reloading: ", True, (0, 0, 0))
            self.screen.blit(text, (self.statx, self.staty + 60))
            pygame.draw.line(self.screen, RED, (self.statx + text.get_width(), self.staty + text.get_height()/2 + 60), (self.statx + text.get_width() + self.reloadtime, self.staty + text.get_height()/2 + 60), 10)

        text = font.render("Bullets left: " + str(round(self.bulletsleft)), True, (0, 0, 0))
        self.screen.blit(text, (self.statx, self.staty + 40))

        text = font.render("Fuel left: ", True, (0, 0, 0))
        self.screen.blit(text, (self.statx, self.staty + 20))
        if self.fuel > 0:
            pygame.draw.line(self.screen, self.fuellinecolor, (self.statx + text.get_width(), text.get_height() / 2 + self.staty + 20),
                            (self.statx + text.get_width() + self.fuel / 33.3, text.get_height() / 2 + self.staty + 20), 10)

        text = font.render("Score: " + str(round(self.score)), True, (0, 0, 0))
        self.screen.blit(text, (self.statx, self.staty))

    def update(self):
        """
        Обработка всех остальных событий связанных с танком
        Отрисовка пушки с заданным углом и размером.
        Перезарядка пушки и отрисовка количества оставшихся патронов и таймера перезарядки.
        Создание эффектов выхлопных газов.
        Обновление и отрисовка количества оставшегося топлива.
        Перемещение танка на экране
        """
        self.timer = (self.timer + 1)%1000

        if self.controltype == 1:
            if self.anup:
                self.an += 0.03
            if self.andown:
                self.an -= 0.03

        if 0 > self.an:
            self.an = 0
        if math.pi < self.an:
            self.an = math.pi

        self.sin = math.sin(self.an)
        self.cos = math.cos(self.an)
        self.gunxcoord = self.x + (30 + self.f2_power)*self.cos
        self.gunycoord = self.y - (30 + self.f2_power)*self.sin
        pygame.draw.polygon(self.screen, BLACK, ((self.x - 5*self.sin, self.y-5*self.cos), (self.x + 5*self.sin, self.y + 5*self.cos),
                                                 (self.gunxcoord + 5*self.sin, self.gunycoord + 5*self.cos),
                                                 (self.gunxcoord - 5*self.sin, self.gunycoord - 5*self.cos)))

        draw_tank(self.screen, self.x - 75, self.y - 85, 0.5, self.color)

        if self.reloading:
            self.reloadtime -= 1
        if self.reloadtime == 0:
            self.reloading = False
            self.reloadtime = self.defreloadtime
            self.bulletsleft = self.defbulletsleft

        self.drawstats()

        if self.fuel > 0:
            if self.moving_left:
                self.vx = -self.maxspeed

            if self.moving_right:
                self.vx = self.maxspeed

            if (self.moving_left or self.moving_right) and self.timer % 2 == 0:
                new_particle = Particle(self.x - 60, self.y + 8, 40, 4, self.screen)
                particles.append(new_particle)
                self.fuel -= 5

            elif self.timer % 5 == 0:
                new_particle = Particle(self.x - 60, self.y + 8, 40, 2, self.screen)
                particles.append(new_particle)
                self.fuel -= 2

            self.x += self.vx
            if abs(self.vx) > 1:
                self.fuellinecolor = RED
                self.vx -= 0.5 * self.vx / abs(self.vx)
            else:
                self.fuellinecolor = GREEN
                self.vx = 0
        else:
            self.fuel = 0

        self.power_up()

    def move(self, event):
        """
        Управление танком с клавиатуры, перемещение вправо, влево, перезарядка
        """
        if event.type == pygame.KEYDOWN:
            if event.key == self.k_move_backward:
                self.moving_left = True
            if event.key == self.k_move_forward:
                self.moving_right = True
            if event.key == self.k_reload and self.bulletsleft != self.defbulletsleft:
                self.bulletsleft = 0
                self.reloading = True
            if event.key == self.changebullettype:
                self.bullettype = (self.bullettype + 1)%2
        if event.type == pygame.KEYUP:
            if event.key == self.k_move_backward:
                self.moving_left = False
            if event.key == self.k_move_forward:
                self.moving_right = False

        if self.controltype == 1:
            if event.type == pygame.KEYDOWN:
                if event.key == self.k_fire:
                    self.fire2_start()
            if event.type == pygame.KEYUP:
                if event.key == self.k_fire:
                    self.fire2_end(self.bullettype)

        if self.controltype == 0:
            if event.type == pygame.MOUSEBUTTONDOWN:
                self.fire2_start()
            if event.type == pygame.MOUSEBUTTONUP:
                self.fire2_end()

    def power_up(self):
        # Увеличение мощности пушки при зажатой ЛКМ.
        if self.f2_power < 50 and self.f2_on:
            self.f2_power += 1

# Класс для мишеней
class Target:
    def __init__(self, _screen, _movingtype = 0):
        """
        :param _screen: Поверхность для отрисовки
        :param _movingtype: Тип движения мишени, 0  если не движется, 1 если движется вправо влево, 2 вверх вниз
        """
        self.screen = _screen
        self.movingtype = _movingtype

        self.live = True
        self.defcooldown = self.cooldown = 90

        x = self.x = random.randint(200,HEIGHT - 50)
        y = self.y = random.randint(200, WIDTH - 200)
        r = self.r = random.randint(10, 30)
        color = self.color = RED
        self.reward = 60 - self.r

        if self.movingtype == 1:
            self.startpoint = self.x
            self.endpoint = self.x + random.randint(-200, -50)
            self.speed = random.randint(1, 4)

        if self.movingtype == 2:
            self.startpoint = self.y
            self.endpoint = self.y + random.randint(-200, -50)
            self.speed = random.randint(1, 4)

        if self.movingtype == 3:
            self.vx = 0
            self.vy = 0

    def new_target(self):
        """ Инициализация новой цели. """
        x = self.x = random.randint(200, HEIGHT - 50)
        y = self.y = random.randint(200, WIDTH - 200)
        r = self.r = random.randint(10, 30)
        color = self.color = RED
        self.reward = 60 - self.r

        if self.movingtype == 1:
            self.startpoint = self.x
            self.endpoint = self.x + random.randint(-200, -50)
            self.speed = random.randint(1, 4)

        if self.movingtype == 2:
            self.startpoint = self.y
            self.endpoint = self.y + random.randint(-200, -50)
            self.speed = random.randint(1, 4)

    def hit(self):
        """Попадание шарика в цель. Возвращает количество очков при попадании"""
        if self.live:
            self.new_target()
            self.live = False
            return self.reward
        else:
            return 0

    def draw(self):
        # Отрисовка мишени
        pygame.draw.circle(self.screen, self.color, (self.x, self.y), self.r)

    def update(self):
        # Обновленеие мишени (таймер до возродения и движение мишени)
        if not self.live:
            if self.cooldown > 0:
                self.cooldown -= 1
            else:
                self.cooldown = self.defcooldown
                self.live = True
        else:
            self.draw()
        if self.movingtype == 1:
            self.x += self.speed
            if self.x < self.endpoint or self.x > self.startpoint:
                self.speed *= -1

        if self.movingtype == 2:
            self.y += self.speed
            if self.y < self.endpoint or self.y > self.startpoint:
                self.speed *= -1

        if self.movingtype == 3:
            self.x += self.vx
            self.y -= self.vy

            if self.x > WIDTH - self.r and self.vx > 0:
                self.vx *= -0.8
                self.vy *= 0.8
                self.x = WIDTH - self.r
            if self.y > HEIGHT - self.r and self.vy < 0:
                self.vy *= -0.8
                self.vx *= 0.8
                self.y = HEIGHT - self.r
            if self.x < self.r and self.vx < 0:
                self.vx *= -0.8
                self.vy *= 0.8
                self.x = self.r
            self.vy -= 1

    def collisioncheck(self, obj):
        if self.movingtype == 3:
            dist = (self.x - obj.x) ** 2 + (self.y - obj.y) ** 2 - (self.r + obj.r) ** 2
            if dist < 0:
                if obj.movingtype == 0:
                    objvx = 0
                    objvy = 0
                if obj.movingtype == 1:
                    objvx = obj.speed
                    objvy = 0
                if obj.movingtype == 2:
                    objvx = 0
                    objvy = obj.speed
                if obj.movingtype == 3:
                    objvx = obj.vx
                    objvy = obj.vy
                p1 = complex(self.x, self.y)
                p2 = complex(obj.x, obj.y)
                v1 = complex(self.vx, self.vy)
                v2 = complex(objvx, objvy)
                p12 = p1 - p2
                d = ((v1 - v2) / p12).real * p12
                self.vx = (v1 - d).real
                self.vy = (v1 - d).imag
                if obj.movingtype == 3:
                    obj.vx = (v2 + d).real
                    obj.vy = (v2 + d).imag

class TargetSpawner:
    def __init__(self, _screen):
        self.x = 0
        self.y = 150
        self.vx = 5
        self.screen = _screen
        self.cooldown = self.defcooldown = 60

    def draw(self):
        pygame.draw.ellipse(self.screen, GREY, (self.x - 50, self.y - 20, 100, 40))

    def update(self):
        self.x += self.vx
        self.draw()
        if self.cooldown < 0 and len(targets) < 10:
            self.cooldown = self.defcooldown
            new_target = Target(self.screen, 3)
            new_target.x = self.x
            new_target.y = self.y
            new_target.vx = self.vx
            targets.append(new_target)
        if len(targets) < 10:
            self.cooldown -= 1
        if self.x > WIDTH - 50 and self.vx > 0:
            self.vx *= -1
        if self.x < 50 and self.vx < 0:
            self.vx *= -1

resetbutton = Button(screen, 690, 10, 100, 50, 'reset', 1)
helpbutton = Button(screen, 690, 70, 100, 50, 'help', 1)

# Сброс игры до начального положения
def new_game():
    global bullet
    global balls
    global particles
    global clock
    global gun
    global targets
    global targetspawner
    bullet = 0
    balls = []
    particles = []
    clock = pygame.time.Clock()
    gun = [Gun(screen, 1) for i in range(2)]
    targets = [Target(screen, i % 3) for i in range(6)]
    targetspawner = TargetSpawner(screen)

    gun[0].name = "Player 1"

    gun[1].setgunstats(pygame.K_RIGHT, pygame.K_LEFT, pygame.K_UP, pygame.K_DOWN, pygame.K_RSHIFT, pygame.K_RETURN, pygame.K_RCTRL, 300, 50)
    gun[1].name = "Player 2"
    gun[1].x = 600
    gun[1].an = math.pi - 1
    gun[1].color = (120, 120, 30)

new_game()
finished = False
while not finished:
    clock.tick(FPS)
    pygame.draw.rect(screen, WHITE, (0, 0, WIDTH, HEIGHT))
    for g in gun:
        g.update()

    resetbutton.draw()
    helpbutton.draw()
    targetspawner.update()

    for p in particles:
        p.update()
        if p.alpha < 0:
            particles.remove(p)
    for t in targets:
        t.update()
        #for t2 in targets:
        #   if t != t2:
        #      t.collisioncheck(t2)
        if t.movingtype == 3 and not t.live:
            targets.remove(t)
    for b in balls:
        b.update()
        for t in targets:
            if t.live:
                if b.hittest(t):
                    b.parent.score += t.hit()
        for b2 in balls:
            if b2 != b:
                b.hittest(b2)
        if b.alpha == 0:
            balls.remove(b)
        b.draw()

    pygame.display.update()

    for event in pygame.event.get():
        if event.type == pygame.QUIT:
            finished = True
        elif resetbutton.check(event):
            new_game()
        elif helpbutton.check(event):
            help()
        elif not resetbutton.pressed:
            for g in gun:
                g.targetting(event)
                g.move(event)

pygame.quit()


import csv
from matplotlib import pyplot as plt
import matplotlib.ticker as ticker
import sys
import math

fname = sys.argv[1]
data = []
with open(fname) as f:
    reader = csv.reader(f)
    headers = next(reader)
    for header in headers:
        data.append([])
    for line in reader:
        for n, val in enumerate(line):
            data[n].append(float(val))

#print(data)
x = data[0]
ys = data[1:]
fftnames = headers[1:]

ys_norm = []
for y in ys:
    ynorm = [1000*y/(n*math.log(n)) for n, y in zip(x,y)]
    ys_norm.append(ynorm)

y_combo = [min((a,b,c)) for a, b, c in zip(ys[0], ys[2], ys[3])]

y_div12 = [y1/y2 for y1, y2 in zip(ys[0], ys[1])]

y_div32 = [y3/y2 for y3, y2 in zip(ys[2], ys[1])]

y_div42 = [y4/y2 for y4, y2 in zip(ys[3], ys[1])]

y_div31 = [y3/y1 for y3, y1 in zip(ys[2], ys[0])]

y_div41 = [y4/y1 for y4, y1 in zip(ys[3], ys[0])]

y_divcombo = [yc/yfftw for yc, yfftw in zip(y_combo, ys[1])]

fig1 = plt.figure(1, figsize=(10,7))
for y in ys:
    plt.plot(x,y, linestyle='dashed', marker=".")
plt.legend(fftnames)
plt.xlabel("Length")
plt.ylabel("Time, us")

fig2 = plt.figure(2, figsize=(10,7))
for y in ys_norm:
    plt.plot(x,y)
plt.legend(fftnames)
plt.xlabel("Length")
plt.ylabel("Time/NlogN")

fig3 = plt.figure(3, figsize=(10,7))
plt.plot(x,y_div12)
plt.grid(True)
fig3.gca().yaxis.set_major_locator(ticker.MultipleLocator(1))
fig3.gca().xaxis.set_major_locator(ticker.MultipleLocator(32))
plt.xlabel("Length")
plt.ylabel("{}/{}".format(fftnames[0], fftnames[1]))

fig4 = plt.figure(4, figsize=(10,7))
plt.plot(x,y_div32)
plt.grid(True)
fig4.gca().yaxis.set_major_locator(ticker.MultipleLocator(1))
fig4.gca().xaxis.set_major_locator(ticker.MultipleLocator(32))
plt.xlabel("Length")
plt.ylabel("{}/{}".format(fftnames[2], fftnames[1]))

fig5 = plt.figure(5, figsize=(10,7))
plt.plot(x,y_div42)
plt.grid(True)
fig5.gca().yaxis.set_major_locator(ticker.MultipleLocator(1))
fig5.gca().xaxis.set_major_locator(ticker.MultipleLocator(32))
plt.xlabel("Length")
plt.ylabel("{}/{}".format(fftnames[3], fftnames[1]))

fig6 = plt.figure(6, figsize=(10,7))
plt.plot(x,y_div31)
plt.grid(True)
fig6.gca().yaxis.set_major_locator(ticker.MultipleLocator(1))
fig6.gca().xaxis.set_major_locator(ticker.MultipleLocator(32))
plt.xlabel("Length")
plt.ylabel("{}/{}".format(fftnames[2], fftnames[0]))

fig7 = plt.figure(7, figsize=(10,7))
plt.plot(x,y_div41)
plt.grid(True)
fig7.gca().yaxis.set_major_locator(ticker.MultipleLocator(1))
fig7.gca().xaxis.set_major_locator(ticker.MultipleLocator(32))
plt.xlabel("Length")
plt.ylabel("{}/{}".format(fftnames[3], fftnames[0]))

fig8 = plt.figure(8, figsize=(10,7))
plt.plot(x,y_divcombo)
plt.grid(True)
fig8.gca().yaxis.set_major_locator(ticker.MultipleLocator(1))
fig8.gca().xaxis.set_major_locator(ticker.MultipleLocator(32))
plt.xlabel("Length")
plt.ylabel("Combo/FFTW")

plt.show()
#print(data)