import os
import sys
import time
import torch
import torch.nn.functional as F
from sklearn.metrics import mean_squared_error
from graphrepr.evaluate import test_model
from graphrepr.savingutils import save_configs, save_history, LoggerWrapper
from graphrepr.config import parse_model_config, parse_representation_config, parse_data_config
from graphrepr.config import utils_section, data_section, params_section, optimizer_section

from graphrepr.chemprop.args import ModelArgs
from graphrepr.chemprop.model import MoleculeModel
from graphrepr.chemprop.features import set_representation
from graphrepr.main_dmpnn_utils import load_data_chemprop, run_epoch, predict

# run: python main.py simple_nn.cfg esol.cfg simple_repr.cfg /home/abc/results

n_args = 1 + 4  # namefile, architecture_config, data_config, representation_config, main_saving_directory
NUM_WORKERS = 2

if __name__ == '__main__':
    if len(sys.argv) != n_args:
        print(f"Usage: {sys.argv[0]} architecture.cfg data.cfg representation.cfg main_saving_directory")
        quit(1)

    # set global saving subdir for this experiment and create it
    # name of the experiment subdir is derived from the names of the configs
    # config name should be: {unique_key}_{whatever}.{ext} ex. 2_best_model.cfg
    basename = lambda x: os.path.basename(x).split('.')[0].split('_')[0]
    dname = "_".join([basename(x) for x in [sys.argv[1], sys.argv[2], sys.argv[3]]])
    saving_dir = os.path.join(sys.argv[4], dname)
    try:
        os.makedirs(saving_dir)
    except FileExistsError:
        pass

    # setup logger (everything that goes through logger or stderr will be saved in a file and sent to stdout)
    logger_wrapper = LoggerWrapper(saving_dir)
    sys.stderr.write = logger_wrapper.log_errors
    logger_wrapper.logger.info(f'Running {[basename(x) for x in [sys.argv[1], sys.argv[2], sys.argv[3]]]}')

    # device selection
    device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')

    # Load configs
    model_config = parse_model_config(sys.argv[1])
    data_config = parse_data_config(sys.argv[2])
    representation_config = parse_representation_config(sys.argv[3])
    save_configs(sys.argv[1], sys.argv[2], sys.argv[3], saving_dir)

    ################
    # # # DATA # # #
    ################
    set_representation(**representation_config[utils_section])

    # define train and validation data splits
    if data_config[utils_section]['cv']:
        trains = []
        vals = []
        for val_fold in data_config[data_section].values():
            trains.append([fold for fold in data_config[data_section].values() if fold != val_fold])
            vals.append([val_fold, ])
        splits = list(zip(trains, vals))
    else:
        trains = [[data_config[data_section][key],] for key in data_config[data_section] if 'train' in key.lower()]
        vals = [[data_config[data_section][key],] for key in data_config[data_section] if 'valid' in key.lower()]

        splits = list(zip(trains, vals))

    # load test
    test_smiles, test_labels, test_loader = load_data_chemprop([data_config[utils_section]["test"], ],
                                                               data_config, model_config,
                                                               shuffle=False, num_workers=NUM_WORKERS)

    #####################
    # # # MAIN LOOP # # #
    #####################
    for fold_idx, (train_paths, validation_path) in enumerate(splits):
        logger_wrapper.logger.info(f'Running fold {fold_idx+1}/{len(splits)}')
        # subdirectory for this fold, file names
        fold_subdirectory = os.path.join(saving_dir, f"fold{fold_idx+1}")
        try:
            os.makedirs(fold_subdirectory)
        except FileExistsError:
            pass
        timestamp = time.strftime('%Y-%m-%d-%H-%M')
        best_model_path = os.path.join(fold_subdirectory, f"{timestamp}-best_model_weights.pt")

        # loading train and validation datasets
        train_dataset, train_smiles, train_loader = load_data_chemprop(train_paths,
                                                                       data_config, model_config,
                                                                       shuffle=True, num_workers=NUM_WORKERS)

        valid_dataset, valid_smiles, valid_loader = load_data_chemprop(validation_path,
                                                                       data_config, model_config,
                                                                       shuffle=False, num_workers=NUM_WORKERS)

        # defining model, optimizer, scheduler, and the loss function
        if "mse" == data_config[utils_section]['cost_function'].lower().strip():
            loss_function = F.mse_loss
            loss_function_valid = mean_squared_error  # workaround cause we have np.arrays not torch.Tensors // FIXME?
            loss_function_model_args = 'mse'
        else:
            raise NotImplementedError("Unknown loss function; only MSE is currently implemented")

        modelArgs = ModelArgs(**model_config[params_section], device=device, loss_function=loss_function_model_args)
        model = MoleculeModel(modelArgs).to(device)
        logger_wrapper.logger.info(model)
        logger_wrapper.logger.info(f"Number of conv layers: {model.encoder.encoder[0].depth}")

        if "adam" == model_config[optimizer_section]['optimizer'].lower().strip():
            optimizer = torch.optim.Adam(model.parameters(), lr=model_config[optimizer_section]['lr'])
        else:
            raise NotImplementedError("Unknown optimizer; only Adam is currently implemented")

        scheduler = None  # for easy checkup later
        if model_config[optimizer_section]['scheduler'] > 0:
            assert 0 < model_config[optimizer_section]['scheduler'] < 1, "scheduler value must be -1 (no scheduler) or between 0 and 1"
            step_size = int(model_config[optimizer_section]['scheduler'] * model_config[optimizer_section]["n_epochs"])
            scheduler = torch.optim.lr_scheduler.StepLR(optimizer, step_size=step_size, gamma=0.1)  # divide lr by ten after every step_size epochs

        # actual training
        train_loss = []
        valid_loss = []
        min_valid_loss = sys.maxsize
        for epoch in range(model_config[optimizer_section]["n_epochs"]):
            # train
            cumulative_epoch_train_loss = run_epoch(model, loss_function, optimizer, train_loader, device)
            # validate
            true_ys, pred_ys = predict(model, valid_loader, device)
            # scheduler
            if scheduler is not None:
                scheduler.step()

            # remember stuff
            epoch_valid_loss = loss_function_valid(pred_ys, true_ys)
            train_loss.append(cumulative_epoch_train_loss / len(train_dataset))
            valid_loss.append(epoch_valid_loss)

            logger_wrapper.logger.info(f'Epoch: {epoch}, train loss: {train_loss[-1]}, valid loss: {epoch_valid_loss}')

            if epoch_valid_loss < min_valid_loss:
                logger_wrapper.logger.info("Saving model")
                torch.save(model.state_dict(), best_model_path)
                min_valid_loss = epoch_valid_loss

        save_history(train_loss, valid_loss, fold_subdirectory)

        # testing on the test set
        # load the best version of the model, then repack data and run the test function
        model.load_state_dict(torch.load(best_model_path))
        model.eval()  # set dropout and batch normalization layers to evaluation mode before running inference

        # train gets new loader without shuffling so the order of smiles is OK # FIXME this is not ideal
        train_dataset, train_smiles, train_loader = load_data_chemprop(train_paths,
                                                                       data_config, model_config,
                                                                       shuffle=False, num_workers=NUM_WORKERS)
        data = ((train_loader, train_smiles), (valid_loader, valid_smiles), (test_loader, test_smiles))
        test_model(model, data, device, fold_subdirectory,
                   calculate_parity=data_config[utils_section]["calculate_parity"],
                   calculate_rocauc=data_config[utils_section]["calculate_rocauc"],
                   predict_func=predict)


# This Python code reproduces the upper and low panel of Fig. 6 in Clay et. al. (2008).

#Save the voltage nd time series csv files as v1.csv for the standard HH model (top panel of fig.6 and v2.csv for the revised model (bottom panel for of fig. 6)

from neuron import h
import numpy as np
import matplotlib.pyplot as plt
import timeit
import subprocess
import math

def initialize():
	h.finitialize()
	h.fcurrent()
def integrate():
	#g.begin()

	h.finitialize()
	while h.t<tstop:
		h.fadvance()

cell=h.Section()

nseg=9
cell.nseg=nseg   # number of segments
cell.Ra=35.4 # ohm*cm  # cable resistance
cell.cm=1

l=1# length of the axon in mm
cell.L=l*1000  # length of the axon in um to be read into NEURON
cell.diam=500 # diameter of the axon in um

#insert the mechanism

#cell.insert('kext_clay') #in case the potassium accumulation is used.
cell.insert('hh') #inserting  Clay revised HH model
#cell.insert('hh') # Standard Hodgkin Huxley model
cell.ek = -72
cell.ena = 55

#Stimulation current
stim1=h.IClamp(0,sec=cell)
stim1.delay=100 #ms
stim1.dur=80 #ms
stim1.amp= 250 #nA

#print stim_density * 1000 * area

vec={}
for var in 'i','t','v':
	vec[var]=h.Vector()

vec['t'].record(h._ref_t)

vec['v'].record(cell(0.99)._ref_v)

vec['i'].record(stim1._ref_i)

tstop=200
h.dt=0.01

initialize()
integrate()

np.savetxt('v1.csv', vec['v'], delimiter= ',')
np.savetxt('time.csv', vec['t'], delimiter= ',') # saving the time series

cell2=h.Section()

nseg=9
cell2.nseg=nseg   # number of segments
cell2.Ra=35.4 # ohm*cm  # cable resistance
cell2.cm=1

l=1# length of the axon in mm
cell2.L=l*1000  # length of the axon in um to be read into NEURON
cell2.diam=500 # diameter of the axon in um

#insert the mechanism

cell2.insert('hhrx_clay_2') #inserting  Clay revised HH model

cell2.ek = -72
cell2.ena = 55

vec['v2'] = h.Vector()
vec['v2'].record(cell2(0.99)._ref_v)

#Stimulation current
stim1=h.IClamp(0,sec=cell2)
stim1.delay=100 #ms
stim1.dur=80 #ms
stim1.amp= 250 #nA

tstop=200
h.dt=0.01

initialize()
integrate()

np.savetxt('v2.csv', vec['v2'], delimiter= ',')

## code for plotting the results

v1 = np.genfromtxt('v1.csv',delimiter=',')
v2 = np.genfromtxt('v2.csv',delimiter=',')
time = np.genfromtxt('time.csv', delimiter= ',')

fig = plt.figure()
ax = fig.add_subplot(2,1,1)
plt.plot(time,v1)
plt.xlabel( 'Time (ms)')
plt.ylabel(' Voltage (mV)')
plt.text(140,20, 'HH', fontsize = 12)
plt.xlim(80,200)
ax.set_xticklabels([])
ax.set_xticks([])
ax.set_yticklabels([])
ax.set_yticks([])
plt.ylim(-75,60)
plt.plot((100,180), (50,50), color = 'k', linewidth = 2)
plt.plot((185,185), (-50,0), color = 'k', linewidth = 2)
plt.text(187,-50, '-50', fontsize = 12)
plt.text(187,0, '0 mV', fontsize = 12)

ax = fig.add_subplot(2,1,2)
plt.plot(time,v2)
plt.xlim(80,200)
plt.xlabel( 'Time (ms)')
plt.ylabel(' Voltage (mV)')
plt.text(140,20, 'Clay, et. al. (2008)', fontsize = 12)
ax.set_xticklabels([])
ax.set_xticks([])
ax.set_yticklabels([])
ax.set_yticks([])
plt.ylim(-75,60)
plt.plot((185,185), (-50,0), color = 'k', linewidth = 2)
plt.plot((125,175), (0,0), color = 'k', linewidth = 2)
plt.text(187,-50, '-50', fontsize = 12)
plt.text(187,0, '0 mV', fontsize = 12)
plt.text(145,-15, '50 ms', fontsize = 12)

try:
  plt.savefig('clay_mohit.jpeg',dpi=600, format='jpeg', bbox_inches='tight')
except:
  plt.savefig('clay_mohit.jpeg',dpi=600, format='jpeg') # incase bbox_inches failes

plt.show()


# @Author : tony
# @Date   : 2021/5/2
# @Title  : epjb2009 paper practice
# @Dec    : deal with the dataset

import networkx as nx

# deal with the INT dataset
def readINT(dataUrl):
    G = nx.read_gml(dataUrl)
    list = dict()
    edge_list = []
    for id, label in enumerate(G.nodes()):
        list[int(label)] = int(id)
    for (v0, v1) in G.edges:
        print(list[int(v0)], list[int(v1)])
        edge_list.append([list[int(v0)], list[int(v1)]])
    return edge_list

# deal with the INT dataset
def readPB(dataUrl):
    G = nx.read_gml(dataUrl)
    list = dict()
    edge_list = []
    for id, label in enumerate(G.nodes()):
        list[label] = int(id)
        print(id, label)
    for (v0, v1) in G.edges:
        print(v0, v1)
        print(list[v0], list[v1])
        edge_list.append([list[v0], list[v1]])
    return edge_list

# deal with the Grid dataset
def readGrid(dataUrl):
    G = nx.read_gml(dataUrl, label='id')
    list = dict()
    edge_list = []
    for id, label in enumerate(G.nodes()):
        list[int(label)] = int(id)
        print(id, label)
    for (v0, v1) in G.edges:
        print(v0, v1)
        print(list[v0], list[v1])
        edge_list.append([list[v0], list[v1]])
    return edge_list

# save the txt
def save(edgeIdList, fileName):
    f = open(fileName, 'w')
    temp = ''
    for item in edgeIdList:
        temp += str(item[0]) + ' ' + str(item[1])
        temp += '\n'
    f.write(temp)
    f.close()

if __name__ == '__main__':
    # print('------------- SRART Internet-------------')
    # edge_list = readINT('./data gml/INT.gml')
    # save(edge_list, './data gml/Internet.txt')
    # print('------------- Internet　END -------------')

    # print('------------- SRART Political blogs-------------')
    # edge_list = readPB('./data gml/PB.gml')
    # print(edge_list)
    # save(edge_list, './data gml/Political blogs.txt')
    # print('------------- Political blogs　END -------------')

    print('------------- SRART Power grid-------------')
    edge_list = readGrid('./data gml/Grid.gml')
    print(edge_list)
    save(edge_list, 'data gml/Power grid.txt')
    print('------------- Power grid　END -------------')


from fastapi import APIRouter

from app.api.segmentation import doc_parser, paddle
from app.core.config import settings
from app.schemas.doc_parser import ImageInput, ImageOutput

router = APIRouter(prefix="/segment", tags=["segment"])

@router.post("/detect-image")
async def doc_parser_api(*, img_in: ImageInput):
    # print("requesting `doc_parser_api`")
    return await doc_parser.detect_image(img_in)

@router.post("/detect-text")
async def paddle_api(*, img_in: ImageInput):
    # print("requesting `paddle_text_api`")
    return await paddle.detect_text(img_in)

@router.post("/detect-bitmap")
async def paddle_bitmap_api(*, img_in: ImageInput):
    # print("requesting `paddle_bitmap_api`")
    return await paddle.db_detect_bitmap(img_in)

@router.post("/detect-text-fast")
async def paddle_math_api(*, img_in: ImageInput):
    # print("requesting `detect_text_fast`")
    return await paddle.detect_text_fast(img_in)


import argparse

from . import board
from . import constants
from . import solver

def parse_args():
    parser = argparse.ArgumentParser()
    parser.add_argument("board", nargs="?")
    parser.add_argument("-m", "--manual", default=False, action="store_true")
    return parser.parse_args()

def main():
    args = parse_args()

    if not args.board:
        main_board = board.Board(constants.TEST_BOARD)
    else:
        main_board = board.Board(args.board)

    # print(tile.Tile.get((1, 1)).render())
    # print(tile.Tile.get((2, 2)).render())
    # print(main_board.render())

    s = solver.Solver(main_board, manual=args.manual)
    s.solve()

if __name__ == '__main__':
    main()


from selenium import webdriver
from selenium.webdriver.common.keys import Keys

url = ''
driver = webdriver.Chrome()
driver.get(url)

element_founder = driver.find_element_by_name('q')
element_founder.send_keys('selenium')
element_founder.send_keys(Keys.RETURN)

results = driver.find_elements_by_css_selector('h3 > a')
for result in results:
    print(result.text)


import json
import uuid
import websocket
import time
import threading
from parlai.core.params import ParlaiParser

# the socket callback functions operate asynchronously.
# upon exit of a chat, we do not want the user to view any additional messages from the server.
# alas, it is necessary to send two messages ([DONE], and EXIT) in order to fully exist the world pool
# to prevent receiving a message after sending [DONE], we track the user's state with
# this global variable.
RUNNING = True

def _get_rand_id():
    """
    :return: The string of a random id using uuid4
    """
    return str(uuid.uuid4())

def _prBlueBG(text):
    """
    Print given in text with a blue background.

    :param text: The text to be printed
    """
    print("\033[44m{}\033[0m".format(text), sep="")

def on_message(ws, message):
    """
    Prints the incoming message from the server.

    :param ws: a WebSocketApp
    :param message: json with 'text' field to be printed
    """
    if not RUNNING:
        return
    incoming_message = json.loads(message)
    print("\033[0m\n")
    print("Bot: " + incoming_message['text'])
    quick_replies = incoming_message.get('quick_replies')
    if quick_replies is not None and len(quick_replies) > 0:
        print(f"\nOptions: [{'|'.join(quick_replies)}]")
    print("\033[44m\n")

def on_error(ws, error):
    """
    Prints an error, if occurs.

    :param ws: WebSocketApp
    :param error: An error
    """
    print(error)

def on_close(ws):
    """
    Cleanup before closing connection.

    :param ws: WebSocketApp
    """
    # Reset color formatting if necessary
    print("\033[0m")
    print("Connection closed")

def _run(ws, id):
    """
    Takes user input and sends it to a websocket.

    :param ws: websocket.WebSocketApp
    """
    global RUNNING
    while True:
        x = input("\033[44m Me: ")
        print("\033[0m", end="")
        data = {}
        data['id'] = id
        data['text'] = x
        if x == "[DONE]":
            RUNNING = False
        json_data = json.dumps(data)
        ws.send(json_data)
        time.sleep(1)
        if x == "[DONE]":
            time.sleep(1)
            data['text'] = 'EXIT'
            ws.send(json.dumps(data))
            break
    ws.close()

def on_open(ws):
    """
    Starts a new thread that loops, taking user input and sending it to the websocket.

    :param ws: websocket.WebSocketApp that sends messages to a terminal_manager
    """
    id = _get_rand_id()
    threading.Thread(target=_run, args=(ws, id)).start()

def setup_args():
    """
    Set up args, specifically for the port number.

    :return: A parser that parses the port from commandline arguments.
    """
    parser = ParlaiParser(False, False)
    parser_grp = parser.add_argument_group('Terminal Chat')
    parser_grp.add_argument(
        '--port', default=35496, type=int, help='Port to run the terminal chat server'
    )
    parser_grp.add_argument(
        '--host', default='localhost', type=str, help='Host to connect to.'
    )
    return parser.parse_args()

if __name__ == "__main__":
    opt = setup_args()
    port = opt.get('port', 34596)
    host = opt.get('host', 'localhost')
    print("Connecting to port: ", port)
    ws = websocket.WebSocketApp(
        f"ws://{host}:{port}/websocket",
        on_message=on_message,
        on_error=on_error,
        on_close=on_close,
    )
    ws.on_open = on_open
    ws.run_forever()


from django.contrib import admin

from django.contrib.admin.options import TabularInline

from apps.info_section_app.models import SimilarLike, SimilarDislike, SimilarTitle, \
    Favorite, RelatedTitle

class SimilarLikeAdminInLine(TabularInline):
    extra = 1
    model = SimilarLike

class SimilarDislikeAdminInline(TabularInline):
    extra = 1
    model = SimilarDislike

@admin.register(SimilarTitle)
class RestaurantModelAdmin(admin.ModelAdmin):
    inlines = (SimilarDislikeAdminInline, SimilarLikeAdminInLine)

admin.site.register(Favorite)
admin.site.register(RelatedTitle)


import contextlib
import hashlib
import httplib
import socket
import tempfile
import urllib2

try:
    from PIL import Image as PILImage
except ImportError:
    import Image as PILImage  # noqa

from django.conf import settings
from django.core.files import File
from django.db import transaction

from comics.aggregator.exceptions import (
    DownloaderHTTPError, ImageTypeError, ImageIsCorrupt, ImageAlreadyExists,
    ImageIsBlacklisted)
from comics.core.models import Release, Image

# Image types we accept, and the file extension they are saved with
IMAGE_FORMATS = {
    'GIF': '.gif',
    'JPEG': '.jpg',
    'PNG': '.png',
}

class ReleaseDownloader(object):
    def download(self, crawler_release):
        images = self._download_images(crawler_release)
        return self._create_new_release(
            crawler_release.comic, crawler_release.pub_date, images)

    def _download_images(self, crawler_release):
        image_downloader = ImageDownloader(crawler_release)
        return map(image_downloader.download, crawler_release.images)

    @transaction.commit_on_success
    def _create_new_release(self, comic, pub_date, images):
        release = Release(comic=comic, pub_date=pub_date)
        release.save()
        for image in images:
            release.images.add(image)
        return release

class ImageDownloader(object):
    def __init__(self, crawler_release):
        self.crawler_release = crawler_release

    def download(self, crawler_image):
        self.identifier = self.crawler_release.identifier

        with self._download_image(
                crawler_image.url, crawler_image.request_headers
                ) as image_file:
            checksum = self._get_sha256sum(image_file)
            self.identifier = '%s/%s' % (self.identifier, checksum[:6])

            self._check_if_blacklisted(checksum)

            existing_image = self._get_existing_image(
                comic=self.crawler_release.comic,
                has_rerun_releases=self.crawler_release.has_rerun_releases,
                checksum=checksum)
            if existing_image is not None:
                return existing_image

            image = self._validate_image(image_file)

            file_extension = self._get_file_extension(image)
            file_name = self._get_file_name(checksum, file_extension)

            return self._create_new_image(
                comic=self.crawler_release.comic,
                title=crawler_image.title,
                text=crawler_image.text,
                image_file=image_file,
                file_name=file_name,
                checksum=checksum)

    def _download_image(self, url, request_headers):
        try:
            request = urllib2.Request(url, None, request_headers)
            with contextlib.closing(urllib2.urlopen(request)) as http_file:
                temp_file = tempfile.NamedTemporaryFile(suffix='comics')
                temp_file.write(http_file.read())
                temp_file.seek(0)
                return temp_file
        except urllib2.HTTPError as error:
            raise DownloaderHTTPError(self.identifier, error.code)
        except urllib2.URLError as error:
            raise DownloaderHTTPError(self.identifier, error.reason)
        except httplib.BadStatusLine:
            raise DownloaderHTTPError(self.identifier, 'BadStatusLine')
        except socket.error as error:
            raise DownloaderHTTPError(self.identifier, error)

    def _get_sha256sum(self, file_handle):
        original_position = file_handle.tell()
        hash = hashlib.sha256()
        while True:
            data = file_handle.read(8096)
            if not data:
                break
            hash.update(data)
        file_handle.seek(original_position)
        return hash.hexdigest()

    def _check_if_blacklisted(self, checksum):
        if checksum in settings.COMICS_IMAGE_BLACKLIST:
            raise ImageIsBlacklisted(self.identifier)

    def _get_existing_image(self, comic, has_rerun_releases, checksum):
        try:
            image = Image.objects.get(comic=comic, checksum=checksum)
            if image is not None and not has_rerun_releases:
                raise ImageAlreadyExists(self.identifier)
            return image
        except Image.DoesNotExist:
            return None

    def _validate_image(self, image_file):
        try:
            image = PILImage.open(image_file)
            image.load()
            return image
        except IndexError:
            raise ImageIsCorrupt(self.identifier)
        except IOError as error:
            raise ImageIsCorrupt(self.identifier, error.message)

    def _get_file_extension(self, image):
        if image.format not in IMAGE_FORMATS:
            raise ImageTypeError(self.identifier, image.format)
        return IMAGE_FORMATS[image.format]

    def _get_file_name(self, checksum, extension):
        if checksum and extension:
            return '%s%s' % (checksum, extension)

    @transaction.commit_on_success
    def _create_new_image(
            self, comic, title, text, image_file, file_name, checksum):
        image = Image(comic=comic, checksum=checksum)
        image.file.save(file_name, File(image_file))
        if title is not None:
            image.title = title
        if text is not None:
            image.text = text
        image.save()
        return image


# Copyright (c) 2023 Graphcore Ltd. All rights reserved.
# The functional definition in this file was ported to Python
# from XCFun, which is Copyright Ulf Ekström and contributors 2009-2020
# and provided under the Mozilla Public License (v2.0)
# see also:
#   - https://github.com/dftlibs/xcfun
#   - https://github.com/dftlibs/xcfun/blob/master/LICENSE.md

import jax.numpy as jnp
import jax
import numpy as np

def __b88(a, gaa):
        # precompute
        c1 = (4.0 / 3.0)
        c2 = (-8.0 / 3.0)
        c3 = (-3.0 / 4.0) * (6.0 / np.pi) ** (1.0 / 3.0) * 2
        d  = 0.0042
        d2 = d * 2.
        d12 = d *12.

        # actual compute
        log_a     = jnp.log(a/2)
        na43      = jnp.exp(log_a * c1)
        chi2      = gaa / 4* jnp.exp(log_a * c2 )
        chi       = jnp.exp(jnp.log( chi2 ) / 2 )
        b88       = -(d * na43 * chi2) / (1.0 + 6*d * chi * jnp.arcsinh(chi)) *2
        slaterx_a = c3 * na43
        return slaterx_a + b88


""" клиентская часть """

import sys
import json
import time
import re
import logging
import logs.config_client_log
from lib.variables import ACTION, PRESENCE, TIME, USER, ACCOUNT_NAME, RESPONSE, AUTH, ALERT, MSG, ERR200, ERR400, \
    CLIENT_LISTEN, LISTEN, SENDER, MSG, MSG_TEXT, ERROR
from lib.utils import create_socket, server_settings, get_message, send_message
from lib.errors import ReqFieldMissingError, ServerError
from logs.decoration_log import log

CLIENT_LOGGER = logging.getLogger('client')

@log
def message_from_server(message):
    """Функция - обработчик сообщений других пользователей, поступающих с сервера"""
    if ACTION in message and message[ACTION] == MSG and \
            SENDER in message and MSG_TEXT in message:
        print(f'Получено сообщение от пользователя {message[SENDER]}:\n{message[MSG_TEXT]}')
        CLIENT_LOGGER.info(f'Получено сообщение от пользователя {message[SENDER]}:\n{message[MSG_TEXT]}')
    else:
        CLIENT_LOGGER.error(f'Получено некорректное сообщение с сервера: {message}')

@log
def create_message(sock, account_name='Guest'):
    """Функция запрашивает текст сообщения и возвращает его.
    Так же завершает работу при вводе подобной комманды
    """
    message = input('Введите сообщение для отправки или \'exit\' для завершения работы: ')
    if message.lower() == 'exit':
        sock.close()
        CLIENT_LOGGER.info('Завершение работы по команде пользователя.')
        sys.exit(0)
    message_dict = {ACTION: MSG,TIME: time.time(),ACCOUNT_NAME: account_name,MSG_TEXT: message}
    CLIENT_LOGGER.debug(f'Сформирован словарь сообщения: {message_dict}')
    return message_dict

@log
def create_presence(account_name='Guest'):
    '''
    Функция генерирует запрос о присутствии клиента
    :param account_name:
    :return:
    '''
    # {'action': 'presence', 'time': 1573760672.167031, 'user': {'account_name': 'Guest'}}
    out = {ACTION: PRESENCE, TIME: time.time(), USER: {ACCOUNT_NAME: account_name}}
    CLIENT_LOGGER.debug(f'Сформировано {PRESENCE} сообщение для пользователя {account_name}')
    return out

@log
def get_user():
    """
    функция возвращает имя пользователя
    :return:
    """
    while True:
        account = input("введите имя пользователя >>>")
        if not re.match(r"[A-Za-z]", account) or len(account) > 25 or len(account) < 3:
            CLIENT_LOGGER.error(f"недопустимое имя пользователя: {account}")
            print("Имя пользователя должно быть от 3 до 25 латинских символов")
        elif account.lower().strip() == 'guest':
            CLIENT_LOGGER.error(f"недопустимое имя пользователя: {account}")
            print("Недоспустимое имя пользователя")
        else:
            break
    return account

@log
def create_action(account_name, action, msg=None):
    '''
    Функция отдает словарь с текстом сообщения
    :param account_name:
    :return:
    '''
    # {'action': 'presence', 'time': 1573760672.167031, 'user': {'account_name': 'Guest'}}
    out = {ACTION: action, TIME: time.time(), USER: {ACCOUNT_NAME: account_name}, MSG: msg}
    CLIENT_LOGGER.debug(f'Сформировано {ACTION} сообщение для пользователя {account_name}')
    return out

@log
def process_handler(message):
    '''
    Функция разбирает ответ сервера
    :param message:
    :return:
    '''
    CLIENT_LOGGER.debug(f'Разбор приветственного сообщения от сервера: {message}')
    print(f'Разбор приветственного сообщения от сервера: {message}')
    if RESPONSE in message:
        if message[RESPONSE]==200: #message[RESPONSE]==200:
            CLIENT_LOGGER.debug(f"{message[RESPONSE]} содержит {ERR200}")
            return message[MSG] #ERR200
        elif message[RESPONSE]==ERR400: #message[RESPONSE]==400:
            CLIENT_LOGGER.debug(f"{message[RESPONSE]} содержит {ERR400}")
            raise ServerError(f"{ERR400}: {message[ERROR]}")
    raise ReqFieldMissingError(RESPONSE)

@log
def start_client():
    srv_settings = server_settings()
    server_address = srv_settings[0]
    server_port = srv_settings[1]
    client_listen = srv_settings[2]

    print(f"start client on: {server_address}:{server_port} | listen_mode={client_listen}")
    CLIENT_LOGGER.info(f"client started {server_address}:{server_port} | listen_mode={client_listen}")

    try:
        transport = create_socket()
        transport.connect((server_address, server_port))
        send_message(transport, create_presence())
        answer = process_handler(get_message(transport))
        CLIENT_LOGGER.info(f"соединение с сервером {server_address}:{server_port}. Ответ: {answer}")
        print(f"соединение с сервером {server_address}:{server_port}. Ответ: {answer}")

# авторизация
        account_name = get_user()
        CLIENT_LOGGER.info(f"Guest авторизовался как {account_name}")
        CLIENT_LOGGER.debug(f"отправка {AUTH} сообщения на сервер {server_address}:{server_port} от user={account_name}")
        message_to_server = create_action(account_name, action=AUTH, msg=None)
        send_message(transport, message_to_server)
        try:
            answer = process_handler(get_message(transport))
            print(answer)
        except (ValueError, json.JSONDecodeError):
            print(answer)
            CLIENT_LOGGER.error(f"{ERR400}. Не удалось декодировать сообшение от сервера")
            print(f"{ERR400}. Не удалось декодировать сообшение от сервера")

    except json.JSONDecodeError:
        CLIENT_LOGGER.error(f"не удалось декодировать JSON-строку")
        print(f"не удалось декодировать JSON-строку")
        sys.exit(1)
    except ServerError as error:
        CLIENT_LOGGER.error(f"ошибка при установке соединения: {error.text}")
        print(f"ошибка при установке соединения: {error.text}")
        sys.exit(1)
    except ReqFieldMissingError as missing_error:
        CLIENT_LOGGER.error(f"в ответе сервера нет обязательного поля {missing_error.missing_field}")
        sys.exit(1)
    except ConnectionRefusedError:
        CLIENT_LOGGER.critical(f"Не удалось подключиться к серверу {server_address}:{server_port}")
        sys.exit(1)
    else:
        print(f"клиент - в режиме client_listen={client_listen:}")
    while True:
        if not client_listen:
            try:
                send_message(transport, create_message(transport))
            except (ConnectionResetError, ConnectionError, ConnectionAbortedError):
                CLIENT_LOGGER.error(f"соединение с сервером {server_address}:{server_port} потеряно")
                print(f"соединение с сервером {server_address}:{server_port} потеряно")
                sys.exit(1)
        if client_listen:
            try:
                message_from_server(get_message(transport))
            except (ConnectionResetError, ConnectionError, ConnectionAbortedError):
                CLIENT_LOGGER.error(f"соединение с сервером {server_address}:{server_port} потеряно")
                print(f"соединение с сервером {server_address}:{server_port} потеряно")
                sys.exit(1)

if __name__ == '__main__':
    start_client()


#!/usr/bin/env python
# coding: utf-8

# In[6]:

import time
import json
import pandas as pd
import re
import logging
from datetime import date, datetime, timedelta
from selenium import webdriver
from selenium.webdriver.firefox.service import Service
from selenium.webdriver.common.by import By
from webdriver_manager.chrome import ChromeDriverManager
opts=webdriver.ChromeOptions()
opts.headless=False
driver = webdriver.Chrome(ChromeDriverManager().install() ,options=opts)

def date_extract(jobs):
    filtered_data=[]
    for value in jobs:
        if not value:
            filtered_data.append(str(date.today()))
        else:
            for item in value:
                integer_value=int(str(item))
                dt=date.today()-timedelta(integer_value)
                filtered_data.append(str(dt))
    return filtered_data

def get_df_from_dict(jobs):
    df=pd.DataFrame.from_dict(jobs) #Creating DataFrame
    df.head(10)
    df=df.apply(lambda x: x.astype(str).str.lower()) #converting into lowercase to remove redundancy
    df.head()
    #df.skills=[skill.split("\n") for skill in df.skills]
    df.location=[location.split(" ") for location in df.location]
    df=df.dropna()
    return df

def task_scrape():

    jobs={"title":[],
        "company":[],

        "experience":[],
        #"skills":[],
        "date":[],
        "scraper_run_dt_time":[],
        "location":[]
        #"jd_url":[]
        }
    final_df=pd.DataFrame(jobs)
        #This outer loop is for number of pages to be scraped
        #for role in data["job_roles"]:
         #   jobs={key:[] for key in jobs}
            #print("hello",jobs)
    for n in range(1,4):
        driver.get('https://www.naukri.com/software-engineer-software-developer-data-analyst-data-scientist-machine-learning-engineer-hr-manager-project-manager-cloud-architect-full-stack-developer-full-stack-web-developer-big-data-consultant-jobs-'+str(n)+'?k=Software%20Engineer%2C%20Software%20Developer%2C%20Data%20Analyst%2C%20Data%20Scientist%2C%20Machine%20Learning%20Engineer%2C%20HR%20Manager%2C%20Project%20Manager%2C%20Cloud%20Architect%2C%20Full%20Stack%20Developer%2C%20Full%20Stack%20Web%20Developer%2C%20Big%20Data%20Consultant')
        time.sleep(4)
        job_container = driver.find_elements(By.CSS_SELECTOR,".jobTuple.bgWhite.br4.mb-8")
                # scraping the details from webpage
        for job in job_container:
            driver.implicitly_wait(20)
            title=job.find_element(By.CSS_SELECTOR,"a.title.fw500.ellipsis").text
            company=job.find_element(By.CSS_SELECTOR,"a.subTitle.ellipsis.fleft").text
            location=job.find_element(By.CSS_SELECTOR,".fleft.grey-text.br2.placeHolderLi.location").text
            try:
                exp=job.find_element(By.CSS_SELECTOR,".fleft.grey-text.br2.placeHolderLi.experience").text
            except Exception:
                exp="0 yrs"
            #skills=job.find_element(By.CSS_SELECTOR,".tags.has-description").text
            date_string=job.find_element(By.CSS_SELECTOR,"[class^='type br2 fleft']").text
                    # date_string contains strings like 2 day ago,just now,few hours ago
            #jd=job.find_element(By.TAG_NAME,"a").get_attribute("href")
            date=re.findall(r'\d+',date_string) #extracting numbers out of the date_string
            jobs["title"].append(title)
            jobs["company"].append(company)
            jobs["location"].append(location)
            jobs["experience"].append(exp)
            #jobs["skills"].append(skills)
            jobs["date"].append(date)
            #jobs["jd_url"].append(jd)
            jobs["scraper_run_dt_time"].append(datetime.today())
    jobs["date"]=date_extract(jobs["date"])
    try:
        print(type(jobs))
        dataframe=get_df_from_dict(jobs)
        print(type(dataframe))
        dataframe[15:25]
        final_df=final_df.append(dataframe)
    except:
        logging.error("Error in dict_to_df")
    now=datetime.today()
    dt_time=now.strftime("%H%M%S")
    dt=now.strftime("%Y%m%d")
    filename="scraped_"+dt+"_"+dt_time
    final_df.info()
    final_df[40:45]
    final_df.to_csv('{}.csv'.format(filename))
    driver.quit

task_scrape()

# In[ ]:


import pytest
import pytest_spec.basic as basic

class TestErrors:
    def test_zero(parameter_list):
        with pytest.raises(ZeroDivisionError) as e:
            basic.division_by_zero(1)
        assert e.type == ZeroDivisionError
        assert e.typename == "ZeroDivisionError"
        assert str(e.value) == "division by zero"

# 引数を変えて行うtest
def add(a, b):
    if type(a) is not int or type(b) is not int:
        raise TypeError
    return a + b

@pytest.mark.parametrize("a,b,expected", [(1, 2, 3), (4, 5, 9), (10, 23, 33)])
def test_add(a, b, expected):
    assert add(a, b) == expected

# 引数を変えて、いずれも例外を吐くことを確認するtest
@pytest.mark.parametrize("a,b,expected", [("1", 2, 3), (None, 5, 9), (10, [1], 33)])
def test_add_2(a, b, expected):
    with pytest.raises(TypeError):
        add(a, b)


import logging
import threading
import numpy as np
import pandas as pd
from timeit import default_timer as timer
from datetime import datetime, timedelta, timezone
from time import sleep
import time
import matplotlib.pyplot as mpl
from sklearn.preprocessing import StandardScaler
from sklearn.model_selection import ShuffleSplit
from sklearn.metrics import make_scorer, r2_score
from sklearn.model_selection import cross_val_score
from sklearn.neighbors import KNeighborsRegressor
from utils.history import Historical_Caching
import warnings

warnings.filterwarnings('ignore')
logging.basicConfig(format='%(asctime)s - %(name)s - %(levelname)s - %(message)s', level=logging.INFO)
logger = logging.getLogger(__name__)

# Gives a list of timestamps from the start date to the end date
#
# startDate:     The start date as a string xxxx-xx-xx
# endDate:       The end date as a string year-month-day
# period:		 'minute', 'daily', 'weekly', or 'monthly'
# weekends:      True if weekends should be included; false otherwise
# return:        A numpy array of timestamps
def DateRange(startDate, endDate, period='minute', weekends=True):
    # The start and end date
    sd = datetime.fromtimestamp(startDate)
    ed = datetime.fromtimestamp(endDate)
    # print(startDate, endDate)
    # Invalid start and end dates
    if (sd > ed):
        raise ValueError("The start date cannot be later than the end date.")
    # One time period is a day
    if (period == 'minute'):
        prd = timedelta(minutes=1)
    if (period == 'daily'):
        prd = timedelta(1)
    # One prediction per week
    if (period == 'weekly'):
        prd = timedelta(7)
    # one prediction every 30 days ("month")
    if (period == 'monthly'):
        prd = timedelta(30)
    # The final list of timestamp data
    dates = []
    cd = sd
    while (cd <= ed):
        # If weekdays are included or it's a weekday append the current ts
        if (weekends or (cd.date().weekday() != 5 and cd.date().weekday() != 6)):
            dates.append(cd.timestamp())
        # Onto the next period
        cd = cd + prd
    # print(np.array(dates))
    return np.array(dates)

# Given a date, returns the previous day
#
# startDate:     The start date as a datetime object
# weekends:      True if weekends should counted; false otherwise
def DatePrevDay(startDate):
    # One day
    day = timedelta(minutes=1)
    cd = startDate - day
    return cd

# Load data from the CSV file. Note: Some systems are unable
# to give timestamps for dates before 1970. This function may
# fail on such systems.
#
# path:      The path to the file
# return:    A data frame with the parsed timestamps
def ParseData(path):
    # Read the csv file into a dataframe
    df = None
    while df is None:
        try:
            sleep(0.5)
            df = pd.read_csv(path)
        except Exception as e:
            sleep(0.5)
            pass

    df['Timestamp'] = df['Date']
    # Remove any unused columns (axis = 1 specifies fields are columns)
    df = df.drop('Date', axis=1)
    # df = df.iloc[::-1]  # CHECK THIS
    return df

# Given dataframe from ParseData
# plot it to the screen
#
# df:        Dataframe returned from
# p:         The position of the predicted data points
def PlotData(df, p=None):
    if (p is None):
        p = np.array([])
    # Timestamp data
    ts = df.Timestamp.values
    # Number of x tick marks
    nTicks = 10
    # Left most x value
    s = np.min(ts)
    # Right most x value
    e = np.max(ts)
    # Total range of x values
    r = e - s
    # Add some buffer on both sides
    s -= r / 5
    e += r / 5
    # These will be the tick locations on the x axis
    tickMarks = np.arange(s, e, (e - s) / nTicks)
    # Convert timestamps to strings
    strTs = [datetime.fromtimestamp(i).strftime('%Y-%m-%d %H:%M:%S') for i in tickMarks]
    mpl.figure()
    # Plots of the high and low values for the day
    mpl.plot(ts, df.High.values, color='#727272', linewidth=1.618, label='Actual')
    # Predicted data was also provided
    if (len(p) > 0):
        mpl.plot(ts[p], df.High.values[p], color='#7294AA', linewidth=1.618, label='Predicted')
    # Set the tick marks
    mpl.xticks(tickMarks, strTs, rotation='vertical')
    # Set y-axis label
    mpl.ylabel('Crypto Price (USD)')
    # Add the label in the upper left
    mpl.legend(loc='upper left')
    mpl.show()

# A class that predicts stock prices based on historical stock data
class Predictor:
    # Constructor
    # nPrevDays:     The number of past days to include
    #               in a sample.
    # rmodel:        The regressor model to use (sklearn)
    # nPastDays:     The number of past days in each feature
    # scaler:        The scaler object used to scale the data (sklearn)
    def __init__(self, rmodel, nPastDays, scaler=StandardScaler()):
        self.npd = nPastDays
        self.R = rmodel
        self.S = scaler
        self.D = None
        self.D_orig = None
        self.DTS = None
        self.A = None
        self.y = None
        self.targCols = None

    # Extracts features from stock market data
    #
    # D:         A dataframe from ParseData
    # ret:       The data matrix of samples
    def _ExtractFeat(self, D):
        # One row per day of stock data
        m = D.shape[0]
        # Open, High, Low, and Close for past n days + timestamp and volume
        n = self._GetNumFeatures()
        B = np.zeros([m, n])
        # Preserve order of spreadsheet
        for i in range(m - 1, -1, -1):
            self._GetSample(B[i], i, D)
        # Return the internal numpy array
        return B

    # Extracts the target values from stock market data
    #
    # D:         A dataframe from ParseData
    # ret:       The data matrix of targets and the

    def _ExtractTarg(self, D):
        # Timestamp column is not predicted
        tmp = D.drop('Timestamp', axis=1)
        # Return the internal numpy array
        return tmp.values, tmp.columns

    # Get the number of features in the data matrix
    #
    # n:         The number of previous days to include
    #           self.npd is  used if n is None
    # ret:       The number of features in the data matrix
    def _GetNumFeatures(self, n=None):
        if (n is None):
            n = self.npd
        return n * 7 + 1

    # Get the sample for a specific row in the dataframe.
    # A sample consists of the current timestamp and the data from
    # the past n rows of the dataframe
    #
    # r:         The array to fill with data
    # i:         The index of the row for which to build a sample
    # df:        The dataframe to use
    # return;    r
    def _GetSample(self, r, i, df):
        # First value is the timestamp
        r[0] = df['Timestamp'].values[i]
        # The number of columns in df
        n = df.shape[1]
        # The last valid index
        lim = df.shape[0]
        # Each sample contains the past n days of stock data; for non-existing data
        # repeat last available sample
        # Format of row:
        # Timestamp Volume Open[i] High[i] ... Open[i-1] High[i-1]... etc
        for j in range(0, self.npd):
            # Subsequent rows contain older data in the spreadsheet
            ind = i + j + 1
            # If there is no older data, duplicate the oldest available values
            if (ind >= lim):
                ind = lim - 1
            # Add all columns from row[ind]
            for k, c in enumerate(df.columns):
                # + 1 is needed as timestamp is at index 0
                r[k + 1 + n * j] = df[c].values[ind]
        return r

    # Attempts to learn the stock market data
    # given a dataframe taken from ParseData
    #
    # D:         A dataframe from ParseData
    def Learn(self, D):
        # Keep track of the currently learned data
        self.D = D.copy()
        self.D_orig = D.copy()
        # self.S = StandardScaler(with_mean=True, with_std=True)
        # Keep track of old timestamps for indexing
        self.DTS = np.asarray(self.D.Timestamp.values)
        # Scale the data
        self.S.fit(self.D)
        self.D[self.D.columns] = self.S.transform(self.D)
        # Get features from the data frame
        self.A = self._ExtractFeat(self.D)
        # Get the target values and their corresponding column names
        self.y, self.targCols = self._ExtractTarg(self.D)
        # Create the regressor model and fit it
        self.R.fit(self.A, self.y)
        return True

    # Predicts values for each row of the dataframe. Can be used to
    # estimate performance of the model
    #
    # df:            The dataframe for which to make prediction
    # return:        A dataframe containing the predictions
    def PredictDF(self, df):
        # Make a local copy to prevent modifying df
        D = df.copy()
        # Scale the input data like the training data
        D[D.columns] = self.S.transform(D)
        # Get features
        A = self._ExtractFeat(D)
        # Construct a dataframe to contain the predictions
        # Column order was saved earlier
        P = pd.DataFrame(index=range(A.shape[0]), columns=self.targCols)
        # Perform prediction
        P[P.columns] = self.R.predict(A)
        # Add the timestamp (already scaled from above)
        P['Timestamp'] = D['Timestamp'].values
        # Scale the data back to original range
        P[P.columns] = self.S.inverse_transform(P)
        return P

    # Predict the stock price during a specified time
    #
    # startDate:     The start date as a string in yyyy-mm-dd format
    # endDate:       The end date as a string yyyy-mm-dd format
    # period:		 'daily', 'weekly', or 'monthly' for the time period
    #				 between predictions
    # return:        A dataframe containing the predictions or
    def PredictDate(self, startDate, endDate, period='minute'):
        # Create the range of timestamps and reverse them
        ts = DateRange(startDate, endDate, period)[::-1]
        m = ts.shape[0]
        # Prediction is based on data prior to start date
        # Get timestamp of previous day
        prevts_ = datetime.fromtimestamp(ts[-1]) - timedelta(minutes=1)
        prevts = np.asarray(prevts_.timestamp())
        # Test if there is enough data to continue
        try:
            ind = np.where(self.DTS <= prevts)[0][0]
        except IndexError:
            logger.info('Safety ON')
            ind = 0
            pass
        # There is enough data to perform prediction; allocate new data frame
        P = pd.DataFrame(np.zeros((self.D.shape[0], self.D.shape[1])), index=range(self.D.shape[0]), columns=self.D.columns)
        # Add in the timestamp column so that it can be scaled properly
        P.loc[int(m):int(self.D.shape[0]), 'Timestamp'] = self.D.loc[0:(int(self.D.shape[0] - m)), 'Timestamp']
        P.loc[0:int(m - 1), 'Timestamp'] = ts
        for i in range(self.D.shape[0] - m):
            # If the current index does not exist, repeat the last valid data
            curInd = ind + i
            if (curInd >= self.D.shape[0]):
                curInd = curInd - 1
            # Copy over the past data (already scaled)
            P.iloc[int(m + i)] = self.D_orig.xs(int(curInd))
        # for i in range(len(P)):
        #     print(datetime.datetime.fromtimestamp(P.loc[i, 'Timestamp']))
        # Scale the timestamp (other fields are 0)
        self.S.fit(P)
        P[P.columns] = self.S.transform(P)
        P = P[0:int(m * 2)]
        # B is to be the data matrix of features
        B = np.zeros((1, self._GetNumFeatures()))
        # Add extra last entries for past existing data
        # Loop until end date is reached
        # print(P)
        for i in range(m - 1, -1, -1):
            # Create one sample
            self._GetSample(B[0], i, P)
            # Predict the row of the dataframe and save it
            pred = self.R.predict(B).ravel()
            for j, k in zip(self.targCols, pred):
                P.at[i, j] = k
        # Discard extra rows needed for prediction
        # Scale the dataframe back to the original range
        P[P.columns] = self.S.inverse_transform(P)

        '''for i in range(len(P)):
            print(datetime.fromtimestamp(P.loc[i, 'Timestamp']))
        print(P)'''

        '''j = 0
        for i in P.Timestamp:
            print(dt.fromtimestamp(i))
            j += 1
            if j > 10:
                break'''

        # PlotData(P)
        P = P[0:m]
        return P

    # Test the predictors performance and
    # displays results to the screen
    #
    # D:             The dataframe for which to make prediction
    def Performance(self, df=None):
        # If no dataframe is provided, use the currently learned one
        if df is None:
            D = self.D.copy()
        else:
            self.S.fit(df)
            D = self.S.transform(df)
        # Get features from the data frame
        A = self._ExtractFeat(D)
        # Get the target values and their corresponding column names
        y, _ = self._ExtractTarg(D)
        # Begin cross validation
        ss = ShuffleSplit(n_splits=1, test_size=0.1, train_size=0.9, random_state=0)

        for trn, tst in ss.split(A):
            s2 = cross_val_score(self.R, A[tst], y[tst], cv=5, scoring=make_scorer(r2_score), n_jobs=-1)

        if len(s2) > 1:
            return s2.mean()
        elif len(s2) == 1:
            logger.info(str(s2))
            return s2
        else:
            return 0

class ML_Calculus:

    def __init__(self, ws_bmex, rest, instrument, history_count, per_pred, API_key, API_secret):
        self.client = rest
        self.instrument_bmex = instrument
        self.API_key_bmex = API_key
        self.API_secret_bmex = API_secret
        self.periods_pred = per_pred - 1
        self.p_verdict = 0
        self.D = None
        self.ready = False
        self.history = Historical_Caching(ws_bmex, rest, instrument, history_count)
        self.thread = threading.Thread(target=self.Engine)
        self.R = KNeighborsRegressor(n_neighbors=10, weights='distance', algorithm='auto', leaf_size=25, n_jobs=-1)
        self.sp_Classic = Predictor(rmodel=self.R, nPastDays=50)
        self.logger = logging.getLogger(__name__)

    def Main(self, args):
        if (len(args) != 3 and len(args) != 4):
            return
        # Test if file exists
        try:
            open(args[0])
        except Exception as e:
            logger.error('Error opening args: ' + args[0])
            logger.error(str(e))
            return

        if (len(args) == 4):
            predPrd = args[3]
            if predPrd == 'm':
                predPrd = 'minute'
            if predPrd == 'D':
                predPrd = 'daily'
            if predPrd == 'W':
                predPrd = 'weekly'
            if predPrd == 'M':
                predPrd = 'monthly'

        try:
            # Everything looks okay; proceed with program
            # Grab the data frame
            # self.D = pand.DataFrame(index=range(self.hc))
            self.D = None
            self.D = ParseData(args[0])
            # The number of previous days of data used
            # when making a prediction

            # PlotData(D)
            s2_mean = 0
            P = None
            i = 0
            res = 0

            while res < 1:
                self.sp_Classic.Learn(self.D)
                res += 1
            while s2_mean < 0.70 and i < 3:
                # Learn the dataset and then display performance statistics
                # sp.TestPerformance()
                # Perform prediction for a specified date range
                P = self.sp_Classic.PredictDate(args[1], args[2])
                if P is None:
                    logger.info(self.instrument_bmex + ': TYPE 2 Reboot')
                    return 0, 0, 0
                s2_mean = self.sp_Classic.Performance()
                # Keep track of number of predicted results for plot
                # n = P.shape[0]
                # Append the predicted results to the actual results
                # D = P.append(D)
                # Predicted results are the first n rows
                # D.to_csv(r'xbt_m1_treated.csv', index=False)
                # PlotData(D, range(n + 1))'''
                i += 1
            # print(P)
            return i, P, s2_mean
        except Exception as e:
            logger.error(str(e))
            sleep(1)
            return 0, 0, 0

    def Engine(self):
        datetime_minute_cached = None
        fails = 0
        self.history.start_supplychain()
        while self.history.get_data_loaded() is False or self.history.get_run_completed() is False:
            logger.info(self.instrument_bmex + ': Waiting for historical data... ')
            sleep(5)
            continue
        logger.info(self.instrument_bmex + ': Starting machine learning computation...')

        while True:
            try:
                if datetime_minute_cached != datetime.utcnow().minute:  # and self.history.get_run_completed() is True:
                    start_timer = timer()

                    timestamp_ = datetime.now(timezone.utc).strftime('%Y-%m-%d %H:%M:00')

                    timestamp = datetime.strptime(timestamp_, '%Y-%m-%d %H:%M:%S').replace(tzinfo=timezone.utc)
                    timestamp = timestamp.timetuple()
                    start_ts = time.mktime(timestamp)

                    timestamp = datetime.strptime(timestamp_, '%Y-%m-%d %H:%M:%S').replace(tzinfo=timezone.utc) + timedelta(minutes=(self.periods_pred))
                    timestamp = timestamp.timetuple()
                    end_ts = time.mktime(timestamp)

                    naming = str(self.instrument_bmex + "_pair_m1.csv")
                    p = None
                    counter = 0

                    while isinstance(p, pd.DataFrame) is False and counter < 5:
                        i, p, s2_mean = self.Main([naming, start_ts, end_ts, 'm'])
                        counter += 1

                    if counter >= 5:
                        logger.warning(self.instrument_bmex + ': Predictor: Error p format')
                        continue

                    elif s2_mean < 0.70:
                        self.p_verdict = 0
                        logger.info(self.instrument_bmex + ' Processing Time: ' + str(round(timer() - start_timer, 5)))
                        logger.info(self.instrument_bmex + ': Machine learning : UNCONCLUSIVE !')
                        self.ready = True
                        fails += 1
                        datetime_minute_cached = datetime.utcnow().minute
                    # print('shape: ', p.shape)
                    # print(p)
                    # print(datetime.datetime.fromtimestamp(p.loc[0, 'Timestamp']))
                    # print(datetime.datetime.fromtimestamp(p.loc[1, 'Timestamp']))
                    # print(datetime.fromtimestamp(p.loc[self.periods_pred, 'Timestamp']))
                    # print(p)
                    else:
                        if np.isnan(s2_mean):
                            self.p_verdict = 0
                        else:
                            temp = self.periods_pred - 1
                            temp_1 = p.loc[temp + 1, 'Close']
                            temp_2 = p.loc[temp + 1, 'Close']
                            j = 0
                            k = 0
                            while temp >= 0:
                                p_close_tx = p.loc[temp, 'Close']
                                if temp_1 < p_close_tx:
                                    j += 1
                                elif temp_2 > p_close_tx:
                                    k += 1
                                temp_1 = temp_2 = p_close_tx
                                temp -= 1
                            if j >= self.periods_pred:
                                self.p_verdict = -1
                            elif k >= self.periods_pred:
                                self.p_verdict = 1
                            else:
                                self.p_verdict = 0
                        logger.info(self.instrument_bmex + ' Processing Time: ' + str(round(timer() - start_timer, 5)))
                        if self.p_verdict == 0:
                            logger.info(self.instrument_bmex + ' -> Machine learning : NEUTRAL !')
                        elif self.p_verdict > 0:
                            logger.info(self.instrument_bmex + ' -> Machine learning : UP !')
                        else:
                            logger.info(self.instrument_bmex + ' -> Machine learning : DOWN !')
                        self.ready = True
                        datetime_minute_cached = datetime.utcnow().minute
                        fails = 0
                    logger.info(self.instrument_bmex + ' -> Machine learning / Non-gaussian metric : ' + str(round(s2_mean * 100, 2)) + "% (iter: " + str(i) + ")")

                sleep(0.1)
            except Exception as e:
                logger.error(str(e))
                sleep(1)
                pass

    def start_ml(self):
        self.thread.daemon = True
        self.thread.start()

    def get_p_verdict(self):
        if self.p_verdict > 0:
            verdict = 1
        elif self.p_verdict < 0:
            verdict = -1
        else:
            verdict = 0
        return verdict


#Item 1 #############################################################################################################
import hashlib
from random import randint
def cadastrar(nome,senha):
    senhaReal = senha
    senhaHash = (hashlib.md5(senhaReal.encode('utf-8')).hexdigest())
    with open("UsuariosCadastrados.txt", "a") as stream:  # usa "w" para não armazenar mais de 1 palavra
        print(nome, senhaHash, file=stream)

def md5(senha):
    senhaNova = (hashlib.md5(senha.encode('utf-8')).hexdigest())
    return senhaNova

def autenticar(nome,senha):
    if nome in listaUsu:
        z = listaUsu.index(nome)
        if senha == listaUsu[z+1]:
            print("Autentificacao efetuada com sucesso. Bem vindo ", nome)
        else:
            print("Nome ou senha invalidos.")
    else:
        print("Nome ou senha invalidos.")
listaUsu = []
listaHash = []
a = 1
while a == 1:
    nomeCadast = input("Digite um nome (de 4 caracteres) para cadastro: ")
    senhaCadast = input("Digite uma senha (de 4 caracteres) para cadastro: ")
    ativa = 1
    if ativa ==1:
        if (len(nomeCadast) or len(senhaCadast)) >4:
            print("Nome ou Senha digitado esta fora dos limites.")
        else:
            listaUsu.append(nomeCadast)
            listaUsu.append(senhaCadast)
            senhaHash = md5(senhaCadast)
            listaHash.append(senhaHash)
            cadastrar(nomeCadast, senhaCadast)
    a = int(input("Digite (0) se quiser parar de fazer cadastros, ou (1) para continuar: "))

fazer = int(input("Digite (1) se deseja autentificar, ou (0) para sair: "))
if fazer == 1:
    nomeAutent = input("Digite seu nome para Login: ")
    senhaAutent = input("Digite sua senha para Login: ")
    autenticar(nomeAutent,senhaAutent)
print("\n")

##Item 2: quebra md5 ################################################################################################
#import hashlib
import string
import datetime
from datetime import timedelta
def quebraMd5(senha1,senha2,senha3,senha4):
    hashes = [senha1, senha2, senha3, senha4]
    startTime = datetime.datetime.now()
    tempo_comeco = datetime.datetime.now()
    x = 0
    # diminui = timedelta(seconds=1) --> SE QUISER DIMINUIR UM TEMPO DETERMINADO
    for a in string.printable:
        for b in string.printable:
            for c in string.printable:
                for d in string.printable:
                    word = a + b + c + d
                    hash = hashlib.md5(word.encode("utf-8")).hexdigest()
                    # print(str(hash))
                    if hash in hashes:
                        end_time = str(datetime.datetime.now() - startTime).split('.')[0]
                        print("Senha Hash quebrada com sucesso!!")
                        print("Senha original: ", word)
                        print("Codigo HASH: ", hash)
                        print("Tempo necessário: ", end_time)
                        print("\n")
                        startTime = datetime.datetime.now()
                        x = x + 1
                        if x >= len(hashes):
                            tempo_final = str(datetime.datetime.now() - tempo_comeco).split('.')[0]
                            print("Tempo total: ", tempo_final)
                            break

senha1 = listaHash[0]
senha2 = listaHash[1]
senha3 = listaHash[2]
senha4 = listaHash[3]
quebraMd5(senha1,senha2,senha3,senha4)

#Item 3 ##############################################################################################################
def crip(texto,chave): #pode usar numeros tbm
    aq = []
    for cont in range (len(texto)):
        if texto[cont] == (chr(32)) or (chr(97)<=texto[cont]<=chr(122)):
            if texto[cont] == " ":
                aq.append(" ")
            else:
                crip = ord(texto[cont])
                if chr(crip+chave) <= chr(122):
                    crip = chr(crip + chave)
                    aq.append(crip)
                elif chr(crip+chave) > chr(122):
                    numero = (crip + chave)
                    numero = numero - 122
                    crip = chr(96 + numero)
                    aq.append(crip)
        if chr(48)<=texto[cont]<=chr(57):
            crip = ord(texto[cont])
            if chr(crip+chave) <= chr(57):
                crip = chr(crip+chave)
                aq.append(crip)
            elif chr(crip+chave) > chr(57):
                numero = (crip+chave)
                numero = numero - 57
                crip = chr(47 + numero)
                aq.append(crip)
    aq = "".join(aq)
    return(aq)
chaveRandom1 = randint(1,9)
senhaProte1 = crip(senha1,chaveRandom1)
chaveRandom2 = randint(1,9)
senhaProte2 = crip(senha2,chaveRandom2)
chaveRandom3 = randint(1,9)
senhaProte3 = crip(senha3,chaveRandom3)
chaveRandom4 = randint(1,9)
senhaProte4 = crip(senha4,chaveRandom4)
with open("HASHSModificados.txt", "a") as stream:  # usa "w" para não armazenar mais de 1 palavra
    print(senhaProte1, senhaProte2, senhaProte3, senhaProte4, file=stream)
print("\n")
print(f"Novo Hash Protegido de {listaUsu[0]} - {senhaProte1}")
print(f"Novo Hash Protegido de {listaUsu[2]} - {senhaProte2}")
print(f"Novo Hash Protegido de {listaUsu[4]} - {senhaProte3}")
print(f"Novo Hash Protegido de {listaUsu[6]} - {senhaProte4}")


from ipywidgets import Box, HBox, VBox, FloatSlider, FloatProgress, Label, Layout
s1 = FloatSlider(description='Apple', min=-5, max=5, step=0.01, value=0, layout=Layout(width='90%'))
s2 = FloatSlider(description='Horse', min=-5, max=5, step=0.01, value=0, layout=Layout(width='90%'))
s3 = FloatSlider(description='Flower', min=-5, max=5, step=0.01, value=0, layout=Layout(width='90%'))
s4 = FloatSlider(description='Car', min=-5, max=5, step=0.01, value=0, layout=Layout(width='90%'))
p1 = FloatProgress(min=0, max=1, step=0.01, value=0)
p2 = FloatProgress(min=0, max=1, step=0.01, value=0)
p3 = FloatProgress(min=0, max=1, step=0.01, value=0)
p4 = FloatProgress(min=0, max=1, step=0.01, value=0)
l1 = Label(value="0.00")
l2 = Label(value="0.00")
l3 = Label(value="0.00")
l4 = Label(value="0.00")

import numpy as np
values = np.array([0.0, 0.0, 0.0, 0.0])
def softmax(i):
    return np.exp(values[i]) / np.sum(np.exp(values))

def set_values():
    l1.value = "%.2f" % softmax(0)
    l2.value = "%.2f" % softmax(1)
    l3.value = "%.2f" % softmax(2)
    l4.value = "%.2f" % softmax(3)
    p1.value = softmax(0)
    p2.value = softmax(1)
    p3.value = softmax(2)
    p4.value = softmax(3)

def on_value_change(change):
    if change.owner == s1:
        values[0] = change.new
    if change.owner == s2:
        values[1] = change.new
    if change.owner == s3:
        values[2] = change.new
    if change.owner == s4:
        values[3] = change.new
    set_values()

s1.observe(on_value_change, names='value')
s2.observe(on_value_change, names='value')
s3.observe(on_value_change, names='value')
s4.observe(on_value_change, names='value')

def main():
	set_values()
	left_box = VBox([s1, s2, s3, s4], layout=Layout(width='50%'))
	middle_box = VBox([p1, p2, p3, p4])
	right_box = VBox([l1, l2, l3, l4])
	return HBox([left_box, middle_box, right_box])

if __name__ == "__main__":
	main()


# -*- coding: utf8 -*-
from django.db import models

class Timer(models.Model):
    """ Модель таймера """

    start_time = models.DateTimeField(
        verbose_name="Время начала",
        null=True,
        blank=True
    )
    end_time = models.DateTimeField(
        verbose_name="Время конца",
        null=True,
        blank=True
    )
    result_time = models.DurationField(
        verbose_name="Время затраченное на задачу",
        null=True,
        blank=True
    )
    task_user = models.ForeignKey(
        'TaskUser',
        on_delete=models.CASCADE,
        related_name='timers_task_user',
        verbose_name='Задачи пользователя'
    )

    def __str__(self):
        return f'Время начала: {self.start_time}. Время конца: {self.end_time}'

    class Meta:
        db_table = 'timers'
        app_label = 'models_app'
        verbose_name = 'Таймер'
        verbose_name_plural = 'Таймеры'


#!/usr/bin/python3
# Script responsible for removing extra tags of nightly images
# QUAY_ACCESS_TOKEN is needed to set as environment variable before executing script
# The access token is used for authentication against the quay api.

import os
import json
import requests
from dateutil.relativedelta import *
from dateutil.easter import *
from dateutil.rrule import *
from dateutil.parser import *
from datetime import *
import argparse

try:
    QUAY_ACCESS_TOKEN = os.environ['QUAY_ACCESS_TOKEN']
except KeyError as e:
    print("QUAY_ACCESS_TOKEN environment variable is not set. Please, set it before running the script.")
    exit('Script exiting....')

REGISTRY  = "quay.io"
NAMESPACE = "kiegroup"

IMAGES={"kogito-data-index-nightly","kogito-quarkus-ubi8-nightly",
        "kogito-quarkus-jvm-ubi8-nightly","kogito-quarkus-ubi8-s2i-nightly",
        "kogito-springboot-ubi8-nightly","kogito-springboot-ubi8-s2i-nightly",
        "kogito-jobs-service-nightly","kogito-management-console-nightly",
        "kogito-cloud-operator-nightly"
    }
def get_image_tags(image):
    '''
    Get all the available tags for the image
    :param image: image name whose tags needs to be fetched
    :return: tags: List of a strcut with tagName and lastModified as fields
    '''
    tags = []
    r = requests.get('https://{0}/api/v1/repository/{1}/{2}/tag/?onlyActiveTags=true'.format(REGISTRY,NAMESPACE,image) , headers={'content-type': 'application/json', 'Authorization': 'Bearer ' + QUAY_ACCESS_TOKEN })
    image_metadata= json.loads(r.text)
    num_tags = len(image_metadata['tags'])
    for i in range(num_tags):
        tags.append({
            "tagName" : image_metadata['tags'][i]['name'],
            "lastModified" : parse(image_metadata['tags'][i]['last_modified'])
        })

    return tags

def delete_image_tags(image, tags):
    '''
    Deletes the extra image tags from the repository
    :param image: Image whose tags needs to be deleted
    :param tags: List of struct with `tagName` and `last_modified` as fields for the image that needs to be deleted
    '''
    if len(tags) == 0:
        print("Image {} does not have extra tags that needs to be deleted".format(image))
    else:
        for tag in tags:
            requests.delete('https://{0}/api/v1/repository/{1}/{2}/tag/{3}'.format(REGISTRY,NAMESPACE,image,tag['tagName']) , headers={'content-type': 'application/json', 'Authorization': 'Bearer ' + QUAY_ACCESS_TOKEN })
        print("Successfully deleted {} tags for the image {}".format(len(tags),image))

def get_and_delete_old_tags(image,max_tags):
    '''
    Driver function, calls the `get_image_tags` to get all the available tags for a image
    finds the tags that needs to be deleted and then passes them to `delete_image_tags`
    :param image: image name whose old tags needs to be deleted
    :param max_tags: Number of maximum tags to be kept for the image
    '''
    all_tags = get_image_tags(image)

    all_tags = list(filter(lambda tag: tag["tagName"]!="latest", all_tags))   #Filter out the entry with latest as tagName from the struct list
    all_tags.sort(key=lambda tagInfo: tagInfo.get("lastModified"))  #sorting in ascending order to get oldest tag on top
    delete_tags = []
    if (len(all_tags) - max_tags) > 0:
        delete_tags = all_tags[:len(all_tags) - max_tags]

    delete_image_tags(image,delete_tags)

if __name__ == "__main__":
    parser = argparse.ArgumentParser(description='Removes extra tags from the registry')
    parser.add_argument('--max-tags', dest='max_tags', default=50,type=int, help='Defines the maximum number of tags for the image to be available, defaults to 10')
    args = parser.parse_args()

    for image in IMAGES:
        get_and_delete_old_tags(image,args.max_tags)


import cv2
import time
import mediapipe as mp

cap = cv2.VideoCapture(1)
mp_hands = mp.solutions.hands
hands = mp_hands.Hands()    #hands.py ctrl+left mouse
mp_draw = mp.solutions.drawing_utils

new_frame_time = 0
prev_frame_time = 0
while True:

    ret, frame = cap.read()

    new_frame_time = time.time()
    fps = 1 / (new_frame_time - prev_frame_time)
    prev_frame_time = new_frame_time
    cv2.putText(frame, f'FPS: {int(fps)}', (40, 50), cv2.FONT_HERSHEY_SIMPLEX, 1, (255, 0, 0), 3)
    img_RGB = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
    result = hands.process(img_RGB)
    # print(result.multi_hand_landmarks)
    if result.multi_hand_landmarks:  # landmark will give information about x and y coordinates
        for hand_landmark in result.multi_hand_landmarks:
            for id, landmark in enumerate(hand_landmark.landmark):
                # print(id, landmark)
                # To get the pixel
                height, width, channel = frame.shape
                coordinates_x, coordinates_y = int(landmark.x * width), int(landmark.y * height)
                print(id, coordinates_x, coordinates_y)
            mp_draw.draw_landmarks(frame, hand_landmark, mp_hands.HAND_CONNECTIONS)

    cv2.imshow('Frame', frame)
    if cv2.waitKey(1) & 0xFF == ord('q'):
        break
cap.release()
cv2.destroyAllWindows()


import re
import phonenumbers

from django import forms
from django.utils.translation import ugettext_lazy as _

from kavenegar import *
from sentry import http
from sentry.plugins.bases.notify import NotificationPlugin

import sentry_kavenegar

DEFAULT_REGION = 'IR'
MAX_SMS_LENGTH = 160

def validate_phone(phone):
    try:
        p = phonenumbers.parse(phone, DEFAULT_REGION)
    except phonenumbers.NumberParseException:
        return False
    if not phonenumbers.is_possible_number(p):
        return False
    if not phonenumbers.is_valid_number(p):
        return False
    return True

def clean_phone(phone):
    # This could raise, but should have been checked with validate_phone first
    return phonenumbers.format_number(
        phonenumbers.parse(phone, DEFAULT_REGION),
        phonenumbers.PhoneNumberFormat.E164,
    )

def basic_auth(user, password):
    return 'Basic ' + (user + ':' + password).encode('base64').replace('\n', '')

def split_sms_to(data):
    return set(filter(bool, re.split(r'\s*,\s*|\s+', data)))

class KavenegarConfigurationForm(forms.Form):
    api_key = forms.CharField(label=_('API KEY'), required=True,
                              widget=forms.TextInput(attrs={'class': 'span6'}))
    sms_from = forms.CharField(label=_('SMS FROM'), required=True,
                              widget=forms.TextInput(attrs={'class': 'span6'}))
    sms_to = forms.CharField(label=_('SMS To #s'), required=True,
                             help_text=_('Recipient(s) phone numbers separated by commas or lines'),
                             widget=forms.Textarea(attrs={'placeholder': 'e.g. +98935XXXXXXX, 0912XXXXXXXX'}))

    def clean_sms_to(self):
        data = self.cleaned_data['sms_to']
        phones = split_sms_to(data)
        if len(phones) > 10:
            raise forms.ValidationError('Max of 10 phone numbers, {0} were given.'.format(len(phones)))
        for phone in phones:
            if not validate_phone(phone):
                raise forms.ValidationError('{0} is not a valid phone number.'.format(phone))
        return ','.join(sorted(map(clean_phone, phones)))

    def clean(self):
        # TODO: Ping Kavenegar and check credentials (?)
        return self.cleaned_data

class KavenegarPlugin(NotificationPlugin):
    author = 'Amir Asaran'
    author_url = 'https://github.com/amirasaran/sentry-kavenegar'
    version = sentry_kavenegar.VERSION
    description = 'A plugin for Sentry, which sends SMS notifications via Kavenegar'
    resource_links = (
        ('Documentation', 'https://github.com/amirasaran/sentry-kavenegar/blob/master/README.md'),
        ('Bug Tracker', 'https://github.com/amirasaran/sentry-kavenegar/issues'),
        ('Source', 'https://github.com/amirasaran/sentry-kavenegar'),
        ('Kavenegar', 'https://www.kavenegar.com/'),
    )

    slug = 'kavenegar'
    title = _('Kavenegar (SMS)')
    conf_title = title
    conf_key = 'kavenegar'
    project_conf_form = KavenegarConfigurationForm

    def is_configured(self, project, **kwargs):
        return all([self.get_option(o, project) for o in (
            'api_key', 'sms_to')])

    def get_send_to(self, *args, **kwargs):
        # This doesn't depend on email permission... stuff.
        return True

    def notify_users(self, group, event, **kwargs):
        project = group.project

        body = "Sentry [{0}] {1}: {2}".format(
            project.name.encode("utf-8"),
            event.group.get_level_display().upper().encode("utf-8"),
            event.title.encode("utf-8").splitlines()[0],
        )
        body = body[:MAX_SMS_LENGTH]

        api_key = self.get_option('api_key', project)

        sms_to = self.get_option('sms_to', project)
        sms_from = self.get_option('sms_from', project)
        if not sms_to:
            return
        sms_to = split_sms_to(sms_to)

        instance = KavenegarAPI(api_key)

        errors = []

        for phone in sms_to:
            if not phone:
                continue
            try:
                phone = clean_phone(phone)
                params = {
                    'sender': sms_from,
                    'receptor': phone,
                    'message': body
                }
                instance.sms_send(
                    params
                )
            except Exception as e:
                errors.append(e)

        if errors:
            if len(errors) == 1:
                raise errors[0]

            # TODO: multi-exception
            raise Exception(errors)


# Licensed under a 3-clause BSD style license - see LICENSE.rst
"""Test config.py module."""

# STDLIB
import os

# THIRD-PARTY
import numpy as np
import pytest

# SYNPHOT
from synphot.config import conf as synconf
from synphot.utils import generate_wavelengths

# LOCAL
from stsynphot import config
from stsynphot.stio import get_latest_file, irafconvert

class TestOverwriteSynphot:
    """Test if overwriting ``synphot`` defaults is successful."""
    def setup_class(self):
        # For some reason, this does not automatically execute during testing.
        config.overwrite_synphot_config(config.conf.rootdir)

        self.vegafile = synconf.vega_file

    def test_dirname(self):
        assert self.vegafile.startswith(config.conf.rootdir)

    @pytest.mark.remote_data
    def test_isfile(self):
        if self.vegafile.startswith(('ftp', 'http')):
            # This is the case on Travis CI
            pytest.xfail('Cannot test this over FTP or HTTP')
        else:
            assert os.path.isfile(self.vegafile)

@pytest.mark.remote_data
class TestConfigChanges:
    def setup_class(self):
        self.def_dict = config.getref()

    @pytest.mark.parametrize(
        ('cfgname', 'new_val'),
        [('graphtable', 'mtab$n9i1408hm_tmg.fits'),
         ('comptable', 'mtab$n9i1408im_tmc.fits'),
         ('thermtable', 'mtab$n5k15531m_tmt.fits')])
    def test_tables_area(self, cfgname, new_val):
        # Same as config.conf.cfgname = new_val
        setattr(config.conf, cfgname, new_val)
        assert getattr(config.conf, cfgname) == new_val

        # Reset to default
        config.conf.reset(cfgname)
        old_expanded_val = get_latest_file(irafconvert(
            getattr(config.conf, cfgname)))
        assert old_expanded_val == self.def_dict[cfgname]

    def test_area(self):
        config.conf.area = 1
        assert config.conf.area == 1

        # Reset to default
        config.conf.reset('area')
        assert config.conf.area == self.def_dict['area']

    def test_waveset(self):
        w = generate_wavelengths(
            minwave=3000, maxwave=5000, num=100, log=False)
        config.conf.waveset_array = w[0].value.tolist()
        config.conf.waveset = w[1]
        np.testing.assert_allclose(
            [config.conf.waveset_array[0], config.conf.waveset_array[-1]],
            [3000, 4980])
        assert (config.conf.waveset ==
                'Min: 3000, Max: 5000, Num: 100, Delta: None, Log: False')

        # Reset to default
        config.conf.reset('waveset_array')
        config.conf.reset('waveset')
        np.testing.assert_allclose(
            [config.conf.waveset_array[0], config.conf.waveset_array[-1]],
            [500, 25989.72879567])
        assert config.conf.waveset == self.def_dict['waveset']


import xmltodict
import json

# Loading and parsing the xml file.
with open (r'q1.xml', "r") as xml_file:
    xml_data = xml_file.read()
    print(xml_data)

# Convert xml to json
json_data = json.dumps(xmltodict.parse(xml_data), indent=4)

print(json_data)

with open("output.json", "w") as json_file:
    json_file.write(json_data)


#!/usr/bin/env python3
"""An Implement of an autoencoder with pytorch.
This is the template code for 2020 NIAC https://naic.pcl.ac.cn/.
The code is based on the sample code with tensorflow for 2020 NIAC and it can only run with GPUS.
Note:
    1.This file is used for designing the structure of encoder and decoder.
    2.The neural network structure in this model file is CsiNet, more details about CsiNet can be found in [1].
[1] C. Wen, W. Shih and S. Jin, "Deep Learning for Massive MIMO CSI Feedback", in IEEE Wireless Communications Letters, vol. 7, no. 5, pp. 748-751, Oct. 2018, doi: 10.1109/LWC.2018.2818160.
    3.The output of the encoder must be the bitstream.
"""
import numpy as np
import h5py
import torch
from transformer import  *
from dataloader import *
from loss import *
import os
import torch.nn as nn
import pickle
import random
from copy import deepcopy
from util import *
def save_obj(obj, name ):
    with open( name + '.pkl', 'wb') as f:
        pickle.dump(obj, f, pickle.HIGHEST_PROTOCOL)

def load_obj(name ):
    with open(  name + '.pkl', 'rb') as f:
        return pickle.load(f)
# Parameters for training
torch.backends.cudnn.benchmark=True
np.set_printoptions(suppress=True)

os.environ["CUDA_VISIBLE_DEVICES"] = '0'
use_single_gpu = True  # select whether using single gpu or multiple gpus
torch.manual_seed(1)
batch_size = 16
epochs = 8000
learning_rate = 1e-3
num_workers = 4
print_freq = 3200  # print frequency (default: 60)
# parameters for data
# feedback_bits = 48  # sim2 = 0.26814688715868884, multi2 = 2.3217122101795313, multi_div_sim_2 = 8.658359732535498
feedback_bits = 48
B = 2
size_packet = 100

NUM_RX = 4
NUM_TX = 32
NUM_DELAY = 32
NUM_SAMPLE_TRAIN = 4000

def norm_data(x, num_sample, num_rx, num_tx, num_delay):
    x2 = np.reshape(x, [num_sample, num_rx * num_tx * num_delay * 2])
    x_max = np.max(abs(x2), axis=1)
    x_max = x_max[:,np.newaxis]
    x3 = x2 / x_max / 2.0
    y = np.reshape(x3, [num_sample, num_rx, num_tx, num_delay, 2])
    return y

# Model construction
model = AutoEncoder(feedback_bits=48, dropout=0.1)

# model.encoder.load_state_dict(torch.load('submit_pt/encoder_2.pth.tar')['state_dict'])
# model.decoder.load_state_dict(torch.load('submit_pt/generator_2.pth.tar')['state_dict'])

if use_single_gpu:
    model = model.cuda()
else:
    # DataParallel will divide and allocate batch_size to all available GPUs
    model = torch.nn.DataParallel(model).cuda()

import scipy.io as scio
criterion = SmiLoss()
optimizer = torch.optim.AdamW(model.parameters(), lr=learning_rate)
"""
scheduler = WarmUpCosineAnnealingLR(optimizer=optimizer,
                                            T_max=epochs * len(train_loader),
                                            T_warmup=epochs//20 * len(train_loader),
                                            eta_min=1e-6)
"""

data_train = h5py.File('data/H2_32T4R.mat', 'r')
data_train = np.transpose(data_train['H2_32T4R'][:])
data_train = data_train[:, :, :, :, np.newaxis]
data_train = np.concatenate([data_train['real'], data_train['imag']], 4) # 500 4 32 32 2
# data_train = np.reshape(data_train, [NUM_SAMPLE_TRAIN, NUM_RX* NUM_TX, NUM_DELAY* 2, 1])
# x_train = norm_data(data_train, NUM_SAMPLE_TRAIN, NUM_RX, NUM_TX, NUM_DELAY)
data_train = data_train.astype(np.float32)

x_train = data_train
x_test = data_train

"""
x_test = x_test[1000:,:,:,:]
"""

x_train_hat = np.transpose(x_train, (0,3,1,2,4)).reshape(-1, 32, 256)
x_train_paterns = np.unique((np.sum(np.abs(x_train_hat), axis=2) != 0).astype(np.float32), axis=0)

# dataLoader for training
train_dataset = DatasetFolder(x_train, data_an=True)
print(train_dataset.__len__())
train_loader = torch.utils.data.DataLoader(
    train_dataset, batch_size=batch_size, shuffle=True, pin_memory=True, num_workers=num_workers)

def random_cutmix(x):
    a = x[:,:,:,:16,:]
    b = x[:,:,:,16:,:]
    idx = [i for i in range(a.size(0))]
    random.shuffle(idx)
    res = torch.cat([a,b[idx]], dim=3)
    return res

def random_mixup(x, num=3):
    weight = np.random.randn(num)
    weight = weight / np.sum(weight)
    res = x * weight[0]
    for i in range(1, num):
        idx = [i for i in range(x.size(0))]
        random.shuffle(idx)
        res += x[idx] * weight[i]
    return res

best_loss = 100000
"""
scheduler = WarmUpCosineAnnealingLR(optimizer=optimizer,
                                    T_max=epochs * len(train_loader),
                                    T_warmup=epochs//20 * len(train_loader),
                                    eta_min=1e-6)
"""
print('----', len(train_loader))
for epoch in range(epochs):

    if epoch == 200:
        for param_group in optimizer.param_groups:
            param_group['lr'] = learning_rate * 0.1

    # model training
    model.train()
    total_loss = []
    for i, x in enumerate(train_loader):
        input = x
        input = input.cuda()
        # compute output
        output = model(input)

        loss_list = criterion(output, input, epoch=epoch)
        loss = sum(loss_list[1:])
        total_loss.append([item.detach().cpu().numpy()*input.size(0) for item in loss_list])
        optimizer.zero_grad()
        loss.backward()
        optimizer.step()
        if i % print_freq == 0:
            print('Epoch: [{0}][{1}/{2}]\t'
                  'Loss {loss:.6f}\t'.format(
                epoch, i, len(train_loader), loss=loss.item()))
    # model evaluating
    total_loss = np.sum(np.array(total_loss), axis=0) / len(train_dataset)
    print('train loss:{},  other loss:{}\t'.format(total_loss[1], total_loss[2]))
    model.eval()
    total_loss = []
    totalNMSE = 0
    y_test = []
    count = 0
    if epoch%1==0:
        with torch.no_grad():
            for idx in range(int(4000 / size_packet)):
                x = np.random.randint(2, size=(size_packet,feedback_bits))
                x = torch.from_numpy(x)
                x = x.cuda()
                """
                B,_,_,_ = x.size()
                x_var = torch.mean((x.view(B,126,-1).detach() - 0.5)**2,dim = -1)
                x_sort = torch.sort(-x_var,dim = -1)[1] + torch.arange(B).unsqueeze(-1).to(x_var.device)*126
                x_sort = x_sort.view(-1)
                x = x.view(B*126,128,2)
                input = torch.index_select(x, 0, x_sort).view(B,2,126,128)
                """
                input = x
                output = model.decoder(input)   # bx4x32x32x2

                output = output.detach().cpu().numpy()
                if idx == 0:
                    output_all = output
                else:
                    output_all = np.concatenate([output_all, output], axis=0)

        new_output_all = output_all

        real = x_test[:,:,:,:,0] + x_test[:,:,:,:,1]*1j
        fake = new_output_all[:,:,:,:,0] + new_output_all[:,:,:,:,1]*1j

        sim_1, multi_1, multi_div_sim_1 = K_nearest(real, fake, NUM_RX, NUM_TX, NUM_DELAY, 2)
        print('sim:{}, multi:{}, multi_div_sim_1:{}'.format(sim_1, multi_1, multi_div_sim_1))

        if multi_div_sim_1 < best_loss:
            modelSave2 = './submit_pt/generator_2.pth.tar'
            torch.save({'state_dict': model.decoder.state_dict(), }, modelSave2)
            modelSave2 = './submit_pt/encoder_2.pth.tar'
            torch.save({'state_dict': model.encoder.state_dict(), }, modelSave2)

            print("Model saved")
            best_loss = multi_div_sim_1


#!/usr/bin/env python

import sys
import fitsio
import healpy
import numpy as np
import scipy as sp
import argparse
import matplotlib.pyplot as plt

from scipy.interpolate import interp1d
from picca.data import forest
from picca.data import delta
from picca import io

if __name__ == '__main__':

    parser = argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter)

    parser.add_argument('--plate', type = int, default = None, required=True,
            help = 'Plate of spectrum')

    parser.add_argument('--mjd', type = int, default = None, required=True,
            help = 'Modified Julian Date of spectrum')

    parser.add_argument('--fiberid', type = int, default = None, required=True,
            help = 'fiber of spectrum')

    parser.add_argument('--drq', type = str, default = None, required=True,
            help = 'DRQ file')

    parser.add_argument('--nside', type = int, default = 16, required=False,
            help = 'healpix nside')

    parser.add_argument('--spectrum', type = str, default = None, required=True,
            help = 'data directory for all the spectra')

    parser.add_argument('--no-project', action="store_true", required=False,
            help = 'do not project out continuum fitting modes')

    parser.add_argument('--in-dir',type = str,default=None,required=True,
            help='data directory')

    parser.add_argument('--lambda-min',type = float,default=3600.,required=False,
            help='lower limit on observed wavelength [Angstrom]')

    parser.add_argument('--lambda-max',type = float,default=5500.,required=False,
            help='upper limit on observed wavelength [Angstrom]')

    parser.add_argument('--lambda-rest-min',type = float,default=1040.,required=False,
            help='lower limit on rest frame wavelength [Angstrom]')

    parser.add_argument('--lambda-rest-max',type = float,default=1200.,required=False,
            help='upper limit on rest frame wavelength [Angstrom]')

    parser.add_argument('--rebin',type = int,default=3,required=False,
            help='rebin wavelength grid by combining this number of adjacent pixels (ivar weight)')

    parser.add_argument('--mode',type = str,default='pix',required=False,
            help='open mode: pix, spec, spcframe')

    parser.add_argument('--dla-vac',type = str,default=None,required=False,
            help='dla catalog file')

    parser.add_argument('--dla-mask',type = float,default=0.8,required=False,
            help='lower limit on the DLA transmission. Transmissions below this number are masked')

    parser.add_argument('--mask-file',type = str,default=None,required=False,
            help='Path to file to mask regions in lambda_OBS and lambda_RF. In file each line is: region_name region_min region_max (OBS or RF) [Angstrom]')

    parser.add_argument('--flux-calib',type = str,default=None,required=False,
            help='Path to file to previously produced picca_delta.py file to correct for multiplicative errors in the flux calibration')

    parser.add_argument('--ivar-calib',type = str,default=None,required=False,
            help='Path to previously produced picca_delta.py file to correct for multiplicative errors in the pipeline inverse variance calibration')

    args = parser.parse_args()

    ### forest args
    forest.lmin = np.log10(args.lambda_min)
    forest.lmax = np.log10(args.lambda_max)
    forest.lmin_rest = np.log10(args.lambda_rest_min)
    forest.lmax_rest = np.log10(args.lambda_rest_max)
    forest.rebin = args.rebin
    forest.dll = args.rebin*1e-4
    forest.dla_mask = args.dla_mask

    ### Get Healpy pixel of the given QSO
    objs = {}
    ra,dec,zqso,thid,plate,mjd,fid = io.read_drq(args.drq,0.,1000.,keep_bal=True)
    cut = (plate==args.plate) & (mjd==args.mjd) & (fid==args.fiberid)
    if cut.sum()==0:
        print("Object not in drq")
        sys.exit()
    ra = ra[cut]
    dec = dec[cut]
    zqso = zqso[cut]
    thid = thid[cut]
    plate = plate[cut]
    mjd = mjd[cut]
    fid = fid[cut]
    phi = ra
    th = sp.pi/2.-dec
    pix = healpy.ang2pix(args.nside,th,phi)

    ### Get data
    data = None
    if args.mode == "pix":
        data = io.read_from_pix(args.in_dir,pix[0],thid, ra, dec, zqso, plate, mjd, fid, order=None, log=None)
    elif args.mode in ["spec","corrected-spec"]:
        data = io.read_from_spec(args.in_dir,thid, ra, dec, zqso, plate, mjd, fid, order=None, mode=args.mode,log=None)
    elif args.mode =="spcframe":
        data = io.read_from_spcframe(args.in_dir,thid, ra, dec, zqso, plate, mjd, fid, order=None, mode=args.mode, log=None)
    if data is None:
        print("Object not in in_dir")
        sys.exit()
    else:
        data = data[0]

    ### Correct multiplicative flux calibration
    if (args.flux_calib is not None):
        try:
            vac = fitsio.FITS(args.flux_calib)
            head = vac[1].read_header()

            ll_st = vac[1]['loglam'][:]
            st    = vac[1]['stack'][:]
            w     = (st!=0.)
            forest.correc_flux = interp1d(ll_st[w],st[w],fill_value="extrapolate")
            vac.close()
        except:
            print(" Error while reading flux_calib file {}".format(args.flux_calib))
            sys.exit(1)

    ### Correct multiplicative pipeline inverse variance calibration
    if (args.ivar_calib is not None):
        try:
            vac = fitsio.FITS(args.ivar_calib)
            ll  = vac[2]['LOGLAM'][:]
            eta = vac[2]['ETA'][:]
            forest.correc_ivar = interp1d(ll,eta,fill_value="extrapolate",kind="nearest")
            vac.close()
        except:
            print(" Error while reading ivar_calib file {}".format(args.ivar_calib))
            sys.exit(1)

    ### Get the lines to veto
    usr_mask_obs    = None
    usr_mask_RF     = None
    usr_mask_RF_DLA = None
    if (args.mask_file is not None):
        try:
            usr_mask_obs    = []
            usr_mask_RF     = []
            usr_mask_RF_DLA = []
            with open(args.mask_file, 'r') as f:
                loop = True
                for l in f:
                    if (l[0]=='#'): continue
                    l = l.split()
                    if (l[3]=='OBS'):
                        usr_mask_obs    += [ [float(l[1]),float(l[2])] ]
                    elif (l[3]=='RF'):
                        usr_mask_RF     += [ [float(l[1]),float(l[2])] ]
                    elif (l[3]=='RF_DLA'):
                        usr_mask_RF_DLA += [ [float(l[1]),float(l[2])] ]
                    else:
                        raise
            usr_mask_obs    = np.log10(np.asarray(usr_mask_obs))
            usr_mask_RF     = np.log10(np.asarray(usr_mask_RF))
            usr_mask_RF_DLA = np.log10(np.asarray(usr_mask_RF_DLA))
            if usr_mask_RF_DLA.size==0:
                usr_mask_RF_DLA = None

        except:
            print(" Error while reading mask_file file {}".format(args.mask_file))
            sys.exit(1)

    ### Veto lines
    if not usr_mask_obs is None:
        if ( usr_mask_obs.size+usr_mask_RF.size!=0):
            data.mask(mask_obs=usr_mask_obs , mask_RF=usr_mask_RF)

    ### Correct for DLAs
    if not args.dla_vac is None:
        print("adding dlas")
        dlas = io.read_dlas(args.dla_vac)
        for p in data:
            for d in data[p]:
                if d.thid in dlas:
                    for dla in dlas[d.thid]:
                        data.add_dla(dla[0],dla[1],usr_mask_RF_DLA)

    ### Get delta from picca_delta
    done_delta = None
    f = args.spectrum+"/delta-"+str(pix[0])+".fits.gz"
    hdus = fitsio.FITS(f)
    ds = [delta.from_fitsio(h) for h in hdus[1:]]
    for d in ds:
        if (d.plate==args.plate) and (d.mjd==args.mjd) and (d.fid==args.fiberid):
            d.project()
            done_delta = d
            hdus.close()
            break
    if done_delta is None:
        hdus.close()
        print("Object not in spectrum")
        sys.exit()

    ### Observed l
    plt.errorbar(10**data.ll,data.fl,linewidth=2,color='black')
    plt.errorbar(10**done_delta.ll,done_delta.co,linewidth=4,color='red')
    plt.xlabel(r'$\lambda_{\mathrm{Obs.}} \, [\mathrm{\AA}]$',fontsize=30)
    plt.ylabel(r'$f \, [10^{-19} \mathrm{W \, m^{-2} \, nm^{-1}}]$',fontsize=30)
    plt.grid()
    plt.show()

    ### RF l
    plt.errorbar(10**data.ll/(1.+done_delta.zqso),data.fl,linewidth=4,color='black')
    plt.errorbar(10**done_delta.ll/(1.+done_delta.zqso),done_delta.co,linewidth=4,color='red')
    plt.xlabel(r'$\lambda_{\mathrm{R.F.}} \, [\mathrm{\AA}]$',fontsize=30)
    plt.ylabel(r'$f \, [10^{-19} \mathrm{W \, m^{-2} \, nm^{-1}}]$',fontsize=30)
    plt.grid()
    plt.show()