# -*- coding: utf-8 -*-
"""
Created on Mon Nov  1 10:59:27 2021

@author: Sowmya
"""

import numpy as np
import joblib
from sklearn.model_selection import train_test_split, KFold, RepeatedStratifiedKFold
import tensorflow as tf
from tensorflow.keras.utils import to_categorical
from numpy.random import seed
seed(0)
import tensorflow as tf
tf.random.set_seed(0)

import time
from tensorflow import keras
from tensorflow.keras.models import Sequential
from tensorflow.keras.layers import Dense, Dropout, Permute
from tensorflow.keras.layers import Embedding
from tensorflow.keras.layers import LSTM, BatchNormalization
from tensorflow.keras.models import Model
from tensorflow.keras.layers import Input, Masking
#from keras.utils import plot_model

from tensorflow.keras.layers import Conv1D, MaxPooling1D, Flatten, Activation
from sklearn.model_selection import train_test_split

from tensorflow.keras.regularizers import l2
from tensorflow.keras.layers import Bidirectional
from tensorflow.keras import optimizers
from keras.callbacks import EarlyStopping
from keras.callbacks import ModelCheckpoint

rsp_data, rsp_label = joblib.load(open('Resp_32data_raw.dat', 'rb'))
rsp_data = np.array(rsp_data)
rsp_label = np.array(rsp_label)

emg1_data, emg1_label = joblib.load(open('EMG1_32data_raw.dat', 'rb'))
emg1_data = np.array(emg1_data)
emg1_label = np.array(emg1_label)

emg2_data, emg2_label = joblib.load(open('EMG2_32data_raw.dat', 'rb'))
emg2_data = np.array(emg2_data)
emg2_label = np.array(emg2_label)

eog1_data, eog1_label = joblib.load(open('EOG1_32data_raw.dat', 'rb'))
eog1_data = np.array(eog1_data)
eog1_label = np.array(eog1_label)

eog2_data, eog2_label = joblib.load(open('EOG2_32data_raw.dat', 'rb'))
eog2_data = np.array(eog2_data)
eog2_label = np.array(eog2_label)

gsr_data, gsr_label = joblib.load(open('GSR_32data_raw.dat', 'rb'))
gsr_data = np.array(gsr_data)
gsr_label = np.array(gsr_label)

bvp_data, bvp_label = joblib.load(open('Pleth_32data_raw.dat', 'rb'))
bvp_data = np.array(bvp_data)
bvp_label = np.array(bvp_label)

#tmp_data, tmp_label = joblib.load(open('C:/Users/Sowmya/Desktop/github_experiments/FUSION_DEAP/Tmp_32data_raw.dat', 'rb'))
tmp_data, tmp_label = joblib.load(open('Tmp_32data_raw.dat', 'rb'))
tmp_data = np.array(tmp_data)
tmp_label = np.array(tmp_label)

fusion_data = np.hstack((emg1_data, emg2_data, eog1_data, eog2_data, bvp_data, rsp_data, tmp_data, gsr_data ))

def data_binarizer(ratings, threshold1, threshold2):
	"""binarizes the data below and above the threshold"""
	binarized = []
	for rating in ratings:
		if rating < threshold1:
			binarized.append(0)
		elif rating>= threshold2:
			binarized.append(1)
	return binarized

#convert binarized label (0 and 1) into categorical data- this generates 2 classes
from tensorflow.keras.utils import to_categorical
y_valence = np.array(data_binarizer([el[0] for el in emg1_label],5,5))

#use stratify for split
X_train_ff_val, X_test_ff_val, y_train_ff_val, y_test_ff_val = train_test_split(fusion_data, y_valence, test_size=0.2, random_state=42, stratify=y_valence)
print('X_train_ff_val', X_train_ff_val.shape)
print('X_test_ff_val', X_test_ff_val.shape)

'''
#Create balanced data
#oversample
import imblearn
from imblearn.over_sampling import RandomOverSampler
ros = RandomOverSampler(random_state=0)
X_ff_train_resampled, y_ff_train_resampled = ros.fit_resample(X_train_ff_val, y_train_ff_val)
#X_ff_vald_resampled, y_ff_vald_resampled = ros.fit_resample(X_vald_ff_val, y_vald_ff_val)
X_ff_test_resampled, y_ff_test_resampled = ros.fit_resample(X_test_ff_val, y_test_ff_val)
print("x_ff_train_resampled",X_ff_train_resampled.shape)
#print("x_ff_train_resampled",X_ff_vald_resampled.shape)
print("x_ff_test_resampled",X_ff_test_resampled.shape)

from collections import Counter
 # summarize observations by class labeL
counter = Counter(y_ff_train_resampled)
print('Counter: y_ff_train_resampled',counter)
#counter = Counter(y_ff_vald_resampled)
#print('Counter: y_ff_vald_resampled',counter)
counter = Counter(y_ff_test_resampled)
print('Counter: y_ff_test_resampled',counter)
'''
#convert binarized label (0, 1) into categorical data- this generates 2 classes
#Z1 = np.ravel(y_ff_train_resampled)
Z1 = np.ravel(y_train_ff_val)
y_ff_train_resampled = to_categorical(Z1)
#Z11 = np.ravel(y_ff_vald_resampled)
#y_ff_vald_resampled = to_categorical(Z11)
#y_test
#Z22 = np.ravel(y_ff_test_resampled)
Z22 = np.ravel(y_test_ff_val)
y_ff_test_resampled = to_categorical(Z22)

from sklearn.preprocessing import StandardScaler
sc = StandardScaler()
ff_training_set_scaled = sc.fit_transform(X_train_ff_val)
#ff_vald_set_scaled = sc.transform(X_ff_vald_resampled)
ff_testing_set_scaled = sc.transform( X_test_ff_val)

n_features = 8
#order = F,  fortran like is important for separating 2 features (first 128 columns = 1 feature, next 128 columns -2nd feature), else every alternate column become 1 feature.
x_ff_train = ff_training_set_scaled.reshape(ff_training_set_scaled.shape[0], 8064,n_features,  order='F' )
#x_ff_vald = ff_vald_set_scaled.reshape(ff_vald_set_scaled.shape[0], 128,n_features,  order='F' )
x_ff_test = ff_testing_set_scaled.reshape(ff_testing_set_scaled.shape[0], 8064, n_features, order='F')
print("x_ff_train_reshaped", x_ff_train.shape)
#print("x_ff_vald_reshaped", x_ff_vald.shape)
print("x_ff_test_reshaped", x_ff_test.shape)

start = time.time()
# Nicely formatted time string
def hms_string(sec_elapsed):
    h = int(sec_elapsed / (60 * 60))
    m = int((sec_elapsed % (60 * 60)) / 60)
    s = sec_elapsed % 60
    return "{}:{:>02}:{:>05.2f}".format(h, m, s)

GPUS = ["GPU:0", "GPU:1","GPU:2","GPU:3"] #https://github.com/jeffheaton/present/blob/master/youtube/gpu/keras-dual-gpu.ipynb
#strategy = tf.distribute.MirroredStrategy( GPUS)
strategy = tf.distribute.MirroredStrategy(devices = GPUS, cross_device_ops=tf.distribute.HierarchicalCopyAllReduce())
print('Number of devices: %d' % strategy.num_replicas_in_sync)

with strategy.scope():
#with tf.device('/GPU:0'):

    #this was run with threshold 5 and added extra conv1d layer dropout after FIRST, second and third conv1d layer

    model2 = Sequential()
    model2.add(Conv1D(filters=64, kernel_size=3,  strides=1, activation='relu', input_shape=(x_ff_train.shape[1],x_ff_train.shape[2])))
    model2.add(BatchNormalization())
    #model2.add(Dropout(0.3))
    model2.add(Conv1D(filters=64, kernel_size=3,  strides=1, activation='relu'))
    model2.add(BatchNormalization())
    #model2.add(Dropout(0.3))
    model2.add(Conv1D(filters=128, kernel_size=3, activation='relu'))
    model2.add(BatchNormalization())
    model2.add(Dropout(0.3))
    model2.add(Bidirectional(LSTM(64, return_sequences=True)))
    model2.add(BatchNormalization())
    model2.add(Dropout(0.3))
    model2.add(Bidirectional(LSTM(64)))
    model2.add(BatchNormalization())
    model2.add(Dropout(0.3))
    model2.add(Dense(100, activation='relu'))
    model2.add(BatchNormalization())
    model2.add(Dropout(0.3))
    model2.add(Dense(2, activation='softmax'))
    model2.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])

# patient early stopping
es = EarlyStopping(monitor='val_accuracy', mode='max', verbose=1, patience=50)
mc = ModelCheckpoint('ff8channels_500bs_8064_model2.h5', monitor='accuracy', mode='max', verbose=1, save_best_only=True)
model2.summary()
# fit network
history=model2.fit(x_ff_train, y_ff_train_resampled, epochs=300, batch_size=500, verbose=1, callbacks = [es,mc],validation_data=(x_ff_test, y_ff_test_resampled))

elapsed = time.time()-start
print ('Training time: {elapsed in mins)}', hms_string(elapsed))


from django.shortcuts import render
from . models import Category, Product
from django.http import Http404, HttpResponseRedirect

def index(request):
    return render(request,'api/index.html')

def categories(request):
    category_list = Category.objects.order_by('id')
    return render(request, 'api/category_list.html', {'category_list': category_list})

def detail(request, category_id):
    try:
        c = Category.objects.get(id = category_id)
    except:
        raise Http404("Not found!")
    return render(request,'api/category_details.html', {'category':c})

def products(request, category_id):
    try:
        product_list = Product.objects.filter(category_id = category_id)
    except:
        raise Http404("Not found!")
    return render(request,'api/product_list.html', {'product_list':product_list})

def product_detail(request, product_id):
    try:
        p = Product.objects.get(id = product_id)
    except:
        raise Http404("Not found!")
    return render(request, 'api/product_details.html',{'product' : p})


"""
Issues:
    * Delete con_info and con_transfers when con is closed
    * Should contract also be deleted when its transfered? Prob
    * To do: add a clean up routine based on old cons

    Todo: implement RST for -- will probably need to look at  transfer_request_handlers
"""

import pyp2p.unl
import pyp2p.net
import pyp2p.dht_msg
import logging
import storjnode.storage as storage
from collections import OrderedDict
from btctxstore import BtcTxStore
import tempfile
import time
import json
import hashlib
import sys
import os
import binascii
import struct
from threading import Lock
from twisted.internet import defer

mutex = Lock()

_log = logging.getLogger(__name__)

class RequestDenied(Exception):
    pass

class TransferError(Exception):
    pass

def process_transfers(client):
    _log.debug("In process transfers")

    # Process contract messages.
    try:
        if client.net.dht_node.has_messages():
            for msg in client.net.dht_node.get_messages():
                _log.debug("Processing: " + msg["message"])
                client.protocol(msg["message"])
    except Exception as e:
        _log.debug(e)
        pass

    # Process new connections.
    client.net.synchronize()

    # Raise appropriate async callbacks for errors.
    for con in list(client.con_info):
        if not con.connected:
            # Broken connections.
            for contract_id in list(client.con_info[con]):
                if contract_id in client.defers:
                    e = TransferError("Connection died.")
                    client.defers[contract_id].errback(e)
                    del client.defers[contract_id]

                    if contract_id in client.contracts:
                        del client.contracts[contract_id]

    # Expired handshakes.
    for contract_id in list(client.contracts):
        if contract_id in client.handshake:
            handshake = client.handshake[contract_id]
            elapsed = time.time() - handshake["timestamp"]
            if elapsed >= 30:
                if contract_id in client.defers:
                    e = RequestDenied("Handshake timed out.")
                    client.defers[contract_id].errback(e)
                    del client.defers[contract_id]

    # Todo: cleanup con info and other structures.

    # Process connections.
    for con in client.net:
        _log.debug("In con.")

        # This is an new (or old) connection.
        if con not in client.con_info:
            _log.debug("Con not in con_info")
            continue

        # Connection is not synchronized yet.
        if con.nonce == None:
            _log.debug("Not synced yet")
            continue

        # Wait until there's new transfers to process.
        if not client.is_queued(con):
            # Socket has hung ungracefully.
            duration = time.time() - con.alive
            if duration >= 15.0:
                _log.debug("Ungraceful socket close")
                con.close()
                break

            _log.debug("Not queued: skipping")
            continue

        # Get active contract ID.
        if con not in client.con_transfer:
            _log.debug("Con not in con_Transfer")
            continue

        contract_id = client.con_transfer[con]
        _log.debug("Contract id =")
        _log.debug(contract_id)
        if len(contract_id) < 64:
            remaining = 64 - len(contract_id)
            _log.debug("Blocking = " + str(con.blocking))
            partial = con.recv(remaining)
            _log.debug("Contract id chunk = " + partial)
            _log.debug("Connected = " + str(con.connected))
            if not len(partial):
                _log.debug("Did not receive contract id")
                continue

            client.con_transfer[con] += partial
            _log.debug("Skipping contract id.")
            continue

        # Reached end of transfer queue.
        if contract_id == u"0" * 64:
            _log.debug("Skippng end of queue")
            continue

        # Anything left to do?
        if contract_id not in client.con_info[con]:
            continue
        con_info = client.con_info[con][contract_id]
        if not con_info["remaining"]:
            _log.debug("Skipping remaining.")
            continue

        # Upload.
        contract = client.contracts[contract_id]
        transfer_complete = 0
        if client.net.unl == pyp2p.unl.UNL(value=contract["host_unl"]):
            _log.debug("Uploading: Found our UNL")

            # Send file size.
            if not con_info["file_size"]:
                # Get file size.
                path = storage.manager.find(
                    client.store_config,
                    contract["data_id"]
                )
                file_size = os.path.getsize(path)
                con_info["file_size"] = file_size
                con_info["remaining"] = file_size

                # Marshal file size for network.
                if sys.version_info >= (3, 0, 0):
                    net_file_size = struct.pack(
                        "<20s",
                        str(file_size).encode("ascii")
                    )
                else:
                    net_file_size = struct.pack(
                        "<20s",
                        str(file_size)
                    )

                # Send file size.
                con.send(net_file_size, send_all=1)

            # Get next chunk from file.
            position = con_info["file_size"] - con_info["remaining"]
            data_chunk = client.get_data_chunk(
                contract["data_id"],
                position
            )

            # Upload chunk binary to socket.
            bytes_sent = con.send(data_chunk)
            _log.debug(bytes_sent)
            if bytes_sent:
                con_info["remaining"] -= bytes_sent

            _log.debug("Remaining = ")
            _log.debug(con_info["remaining"])

            # Everything uploaded.
            if not con_info["remaining"]:
                transfer_complete = 1
        else:
            _log.debug("Attempting to download.")

            # Get file size.
            if not con_info["file_size"]:
                file_size_buf = con_info["file_size_buf"]
                if len(file_size_buf) < 20:
                    remaining = 20 - len(file_size_buf)
                    partial = con.recv(remaining)
                    if not len(partial):
                        continue

                    file_size_buf += partial
                    if len(file_size_buf) == 20:
                        file_size, = struct.unpack("<20s", file_size_buf)
                        file_size = int(file_size_buf.rstrip(b"\0"))
                        con_info["file_size"] = file_size
                        con_info["remaining"] = file_size
                    else:
                        continue

            # Download.
            data = con.recv(
                con_info["remaining"],
                encoding="ascii"
            )
            _log.debug(con.connected)

            if len(data):
                con_info["remaining"] -= len(data)
                client.save_data_chunk(contract["data_id"], data)

            _log.debug("Remaining = ")
            _log.debug(con_info["remaining"])

            # When done downloading close con.
            if not con_info["remaining"]:
                # Check download.
                data_id = contract["data_id"]
                temp_path = client.downloading[data_id]
                with open(temp_path, "rw") as shard:
                    # Delete file if it doesn't hash right!
                    found_hash = storage.shard.get_id(shard)
                    if found_hash != data_id:
                        _log.debug(found_hash)
                        _log.debug(data_id)
                        _log.debug("Error: downloaded file doesn't hash right!")
                        os.remove(temp_path)
                        continue

                    # Move shard to storage.
                    storage.manager.add(
                        client.store_config,
                        shard
                    )

                # Remove that we're downloading this.
                del client.downloading[data_id]

                # Ready for a new transfer (if there are any.)
                transfer_complete = 1

        their_unl = client.get_their_unl(contract)
        is_master = client.net.unl.is_master(their_unl)
        _log.debug("Is master = " + str(is_master))
        if transfer_complete:
            # Return async success.
            if contract_id in client.defers:
                # Call any callbacks registered with this defer.
                client.defers[contract_id].callback(client.success_value)
                del client.defers[contract_id]

            if is_master:
                client.queue_next_transfer(con)
            else:
                client.con_transfer[con] = u""

class FileTransfer:
    def __init__(self, net, wif=None, store_config=None, handlers=None):
        # Accept direct connections.
        self.net = net

        # Returned by callbacks.
        self.success_value = ("127.0.0.1", 7777)

        # Used for signing messages.
        self.wallet = BtcTxStore(testnet=True, dryrun=True)
        self.wif = wif or self.wallet.create_key()

        # Where will the data be stored?
        self.store_config = store_config
        assert(len(list(store_config)))

        # Handlers for certain events.
        self.handlers = handlers

        # Start networking.
        if not self.net.is_net_started:
            self.net.start()

        # Dict of data requests.
        self.contracts = {}

        # Dict of defers for contracts.
        self.defers = {}

        # Three-way handshake status for contracts.
        self.handshake = {}

        # All contracts associated with this connection.
        self.con_info = {}

        # File transfer currently active on connection.
        self.con_transfer = {}

        # List of active downloads.
        # (Never try to download multiple copies of the same thing at once.)
        self.downloading = {}

    def get_their_unl(self, contract):
        if self.net.unl == pyp2p.unl.UNL(value=contract["dest_unl"]):
            their_unl = contract["src_unl"]
        else:
            their_unl = contract["dest_unl"]

        return their_unl

    def is_queued(self, con=None):
        if con is not None:
            if con not in self.con_info:
                return 0

        if con is None:
            con_list = list(self.con_info)
        else:
            con_list = [con]

        for con in con_list:
            for contract_id in list(self.con_info[con]):
                con_info = self.con_info[con][contract_id]
                if con_info["remaining"]:
                    return 1

        return 0

    def cleanup_transfers(self, con):
        # Close con - there's nothing left to download.
        if not self.is_queued(con):
            # Cleanup con transfers.
            if con in self.con_transfer:
                del self.con_transfer[con]

            # Cleanup con_info.
            if con in self.con_info:
                del self.con_info[con]

            # Todo: cleanup contract + handshake state.

    def queue_next_transfer(self, con):
        _log.debug("Queing next transfer")
        for contract_id in list(self.con_info[con]):
            con_info = self.con_info[con][contract_id]
            if con_info["remaining"]:
                self.con_transfer[con] = contract_id
                con.send(contract_id, send_all=1)
                return

        # Mark end of transfers.
        self.con_transfer[con] = u"0" * 64

    def is_valid_syn(self, msg):
        # List of expected fields.
        syn_schema = (
            u"status",
            u"direction",
            u"data_id",
            u"file_size",
            u"host_unl",
            u"dest_unl",
            u"src_unl",
            u"signature"
        )

        # Check all fields exist.
        if not all(key in msg for key in syn_schema):
            _log.debug("Missing required key.")
            return 0

        # Check SYN size.
        if len(msg) > 5242880: # 5 MB.
            _log.debug("SYN is too big")
            return 0

        # Check direction is valid.
        direction_tuple = (u"send", u"receive")
        if msg[u"direction"] not in direction_tuple:
            _log.debug("Missing required direction tuple.")
            return 0

        # Check the UNLs are valid.
        unl_tuple = (u"host_unl", u"dest_unl", u"src_unl")
        for unl_key in unl_tuple:
            if not pyp2p.unl.is_valid_unl(msg[unl_key]):
                _log.debug("Invalid UNL for " + unl_key)
                _log.debug(msg[unl_key])
                return 0

        # Check file size.
        file_size_type = type(msg[u"file_size"])
        if sys.version_info >= (3, 0, 0):
            expr = file_size_type != int
        else:
            expr = file_size_type != int and file_size_type != long
        if expr:
            _log.debug("File size validation failed")
            _log.debug(type(msg[u"file_size"]))
            return 0

        # Are we the host?
        if self.net.unl == pyp2p.unl.UNL(value=msg[u"host_unl"]):
            # Then check we have this file.
            path = storage.manager.find(self.store_config, msg[u"data_id"])
            if path is None:
                _log.debug("Failed to find file we're uploading")
                return 0
        else:
            # Do we already have this file?
            path = storage.manager.find(self.store_config, msg[u"data_id"])
            if path is not None:
                _log.debug("Attempting to download file we already have")
                return 0

            # Are we already trying to download this?
            if msg[u"data_id"] in self.downloading:
                _log.debug("We're already trying to download this")
                return 0

        return 1

    def protocol(self, msg):
        msg = json.loads(msg, object_pairs_hook=OrderedDict)

        # Associate TCP con with contract.
        def success_wrapper(self, contract_id, host_unl):
            def success(con):
                with mutex:
                    _log.debug("IN SUCCESS CALLBACK")
                    _log.debug("Success() contract_id = " + str(contract_id))

                    # Associate TCP con with contract.
                    contract = self.contracts[contract_id]
                    file_size = contract["file_size"]

                    # Store con association.
                    if con not in self.con_info:
                        self.con_info[con] = {}

                    # Associate contract with con.
                    if contract_id not in self.con_info[con]:
                        self.con_info[con][contract_id] = {
                            "contract_id": contract_id,
                            "remaining": 350, # Tree fiddy.
                            "file_size": file_size,
                            "file_size_buf": b""
                        }

                    # Record download state.
                    data_id = contract["data_id"]
                    if self.net.unl != pyp2p.unl.UNL(value=host_unl):
                        _log.debug("Success: download")
                        fp, self.downloading[data_id] = tempfile.mkstemp()
                    else:
                        # Set initial upload for this con.
                        _log.debug("Success: upload")

                    # Queue first transfer.
                    their_unl = self.get_their_unl(contract)
                    is_master = self.net.unl.is_master(their_unl)
                    _log.debug("Is master = " + str(is_master))
                    if con not in self.con_transfer:
                        if is_master:
                            # A transfer to queue processing.
                            self.queue_next_transfer(con)
                        else:
                            # A transfer to receive (unknown.)
                            self.con_transfer[con] = u""
                    else:
                        if self.con_transfer[con] == u"0" * 64:
                            if is_master:
                                self.queue_next_transfer(con)
                            else:
                                self.con_transfer[con] = u""
            return success

        # Sanity checking.
        if u"status" not in msg:
            return

        # Accept data request.
        if msg[u"status"] == u"SYN":
            # Check syn is valid.
            if not self.is_valid_syn(msg):
                _log.debug("SYN: invalid syn.")
                return

            # Save contract.
            contract_id = self.contract_id(msg)
            self.save_contract(msg)
            self.handshake[contract_id] = {
                "state": u"SYN-ACK",
                "timestamp": time.time()
            }

            # Create reply.
            reply = OrderedDict({
                u"status": u"SYN-ACK",
                u"syn": msg,
            })

            # Sign reply.
            reply = self.sign_contract(reply)

            # Save reply.
            self.send_msg(reply, msg[u"src_unl"])
            _log.debug("SYN")

        # Confirm accept and make connection if needed.
        if msg[u"status"] == u"SYN-ACK":
            # Valid syn-ack?
            if u"syn" not in msg:
                _log.debug("SYN-ACK: syn not in msg.")
                return

            # Is this a reply to our SYN?
            contract_id = self.contract_id(msg[u"syn"])
            if contract_id not in self.contracts:
                _log.debug("--------------")
                _log.debug(msg)
                _log.debug("--------------")
                _log.debug(self.contracts)
                _log.debug("--------------")
                _log.debug("SYN-ACK: contract not found.")
                return

            # Check syn is valid.
            if not self.is_valid_syn(msg[u"syn"]):
                _log.debug("SYN-ACK: invalid syn.")
                return

            # Did I sign this?
            if not self.is_valid_contract_sig(msg[u"syn"]):
                _log.debug("SYN-ACK: sig is invalid.")
                return

            # Update handshake.
            contract = self.contracts[contract_id]
            self.handshake[contract_id] = {
                "state": u"ACK",
                "timestamp": time.time()
            }

            # Create reply contract.
            reply = OrderedDict({
                u"status": u"ACK",
                u"syn_ack": msg
            })

            # Sign reply.
            reply = self.sign_contract(reply)

            # Try make TCP con.
            self.net.unl.connect(
                contract["dest_unl"],
                {
                    "success": success_wrapper(
                        self,
                        contract_id,
                        contract["host_unl"]
                    )
                },
                force_master=0,
                nonce=contract_id
            )

            # Send reply.
            self.send_msg(reply, msg[u"syn"][u"dest_unl"])
            _log.debug("SYN-ACK")

        if msg[u"status"] == u"ACK":
            # Valid ack.
            if u"syn_ack" not in msg:
                _log.debug("ACK: syn_ack not in msg.")
                return
            if u"syn" not in msg[u"syn_ack"]:
                _log.debug("ACK: syn not in msg.")
                return

            # Is this a reply to our SYN-ACK?
            contract_id = self.contract_id(msg[u"syn_ack"][u"syn"])
            if contract_id not in self.contracts:
                _log.debug("ACK: contract not found.")
                return

            # Did I sign this?
            if not self.is_valid_contract_sig(msg[u"syn_ack"]):
                _log.debug("--------------")
                _log.debug(msg)
                _log.debug("--------------")
                _log.debug(self.contracts)
                _log.debug("--------------")
                _log.debug("ACK: sig is invalid.")
                return

            # Is the syn valid?
            if not self.is_valid_syn(msg[u"syn_ack"][u"syn"]):
                _log.debug("ACK: syn is invalid.")
                return

            # Update handshake.
            contract = self.contracts[contract_id]
            self.handshake[contract_id] = {
                "state": u"ACK",
                "timestamp": time.time()
            }

            # Try make TCP con.
            self.net.unl.connect(
                contract["src_unl"],
                {
                    "success": success_wrapper(
                        self,
                        contract_id,
                        contract["host_unl"]
                    )
                },
                force_master=0,
                nonce=contract_id
            )

            _log.debug("ACK")

    def save_contract(self, contract):
        # Record contract details.
        contract_id = self.contract_id(contract)
        self.contracts[contract_id] = contract

        return contract_id

    def send_msg(self, dict_obj, unl):
        node_id = self.net.unl.deconstruct(unl)["node_id"]
        msg = json.dumps(dict_obj, ensure_ascii=True)
        self.net.dht_node.direct_message(
            node_id,
            msg
        )

    def contract_id(self, contract):
        if sys.version_info >= (3, 0, 0):
            contract = str(contract).encode("ascii")
        else:
            contract = str(contract)

        return hashlib.sha256(contract).hexdigest()

    def sign_contract(self, contract):
        if sys.version_info >= (3, 0, 0):
            msg = str(contract).encode("ascii")
        else:
            msg = str(contract)

        msg = binascii.hexlify(msg).decode("utf-8")
        sig = self.wallet.sign_data(self.wif, msg)

        if sys.version_info >= (3, 0, 0):
            contract[u"signature"] = sig.decode("utf-8")
        else:
            contract[u"signature"] = unicode(sig)

        return contract

    def is_valid_contract_sig(self, contract):
        sig = contract[u"signature"][:]
        del contract[u"signature"]

        if sys.version_info >= (3, 0, 0):
            msg = str(contract).encode("ascii")
        else:
            msg = str(contract)

        msg = binascii.hexlify(msg).decode("utf-8")
        address = self.wallet.get_address(self.wif)

        ret = self.wallet.verify_signature(address, sig, msg)
        contract[u"signature"] = sig[:]

        return ret

    def simple_data_request(self, data_id, node_unl, direction):
        file_size = 0
        if direction == u"send":
            action = u"upload"
        else:
            action = u"download"

        return self.data_request(action, data_id, file_size, node_unl)

    def data_request(self, action, data_id, file_size, node_unl):
        """
        Action = put (upload), get (download.)
        """
        _log.debug("In data request function")

        # Who is hosting this data?
        if action == "upload":
            # We store this data.
            direction = u"send"
            host_unl = self.net.unl.value
            assert(storage.manager.find(self.store_config, data_id) is not None)
        else:
            # They store the data.
            direction = u"receive"
            host_unl = node_unl
            if data_id in self.downloading:
                raise Exception("Already trying to download this.")

        # Encoding.
        if sys.version_info >= (3, 0, 0):
            if type(data_id) == bytes:
                data_id = data_id.decode("utf-8")

            if type(host_unl) == bytes:
                host_unl = host_unl.decode("utf-8")

            if type(node_unl) == bytes:
                node_unl = node_unl.decode("utf-8")
        else:
            if type(data_id) == str:
                data_id = unicode(data_id)

            if type(host_unl) == str:
                host_unl = unicode(host_unl)

            if type(node_unl) == str:
                node_unl = unicode(node_unl)

        # Create contract.
        contract = OrderedDict({
            u"status": u"SYN",
            u"direction": direction,
            u"data_id": data_id,
            u"file_size": file_size,
            u"host_unl": host_unl,
            u"dest_unl": node_unl,
            u"src_unl": self.net.unl.value
        })

        # Sign contract.
        contract = self.sign_contract(contract)

        # Route contract.
        contract_id = self.save_contract(contract)
        self.send_msg(contract, node_unl)
        _log.debug("Sending data request")

        # Update handshake.
        self.handshake[contract_id] = {
            "state": "SYN",
            "timestamp": time.time()
        }

        # For async code.
        d = defer.Deferred()
        self.defers[contract_id] = d

        # Return defer for async code.
        return d

    def remove_file_from_storage(self, data_id):
        storage.manager.remove(self.store_config, data_id)

    def move_file_to_storage(self, path):
        with open(path, "rb") as shard:
            storage.manager.add(self.store_config, shard)
            return {
                "file_size": storage.shard.get_size(shard),
                "data_id": storage.shard.get_id(shard)
            }

    def get_data_chunk(self, data_id, position, chunk_size=1048576):
        path = storage.manager.find(self.store_config, data_id)
        buf = b""
        with open(path, "rb") as fp:
            fp.seek(position, 0)
            buf = fp.read(chunk_size)

            return buf

    def save_data_chunk(self, data_id, chunk):
        _log.debug("Saving data chunk for " + str(data_id))
        _log.debug("of size + " + str(len(chunk)))
        assert(data_id in self.downloading)

        # Find temp file path.
        path = self.downloading[data_id]

        _log.debug(path)
        with open(path, "ab") as fp:
            fp.write(chunk)

if __name__ == "__main__":

    # Alice sample node.
    alice_wallet = BtcTxStore(testnet=True, dryrun=True)
    alice = FileTransfer(
        pyp2p.net.Net(
            net_type="direct",
            node_type="passive",
            nat_type="preserving",
            passive_port=60400,
            dht_node=pyp2p.dht_msg.DHT(),
        ),
        wif=alice_wallet.create_key(),
        store_config={"/home/laurence/Storj/Alice": None}  # FIXME temppath
    )

    # ed980e5ef780d5b9ca1a6200a03302f2a91223044bc63dacc6d9f07eead663ab
    # _log.debug(_log.debug(alice.move_file_to_storage("/home/laurence/Firefox_wallpaper.png")))

    # exit()

    # Bob sample node.
    bob_wallet = BtcTxStore(testnet=True, dryrun=True)
    bob = FileTransfer(
        pyp2p.net.Net(
            net_type="direct",
            node_type="passive",
            nat_type="preserving",
            passive_port=60401,
            dht_node=pyp2p.dht_msg.DHT(),
        ),
        wif=bob_wallet.create_key(),
        store_config={"/home/laurence/Storj/Bob": None}  # FIXME use temppath
    )

    _log.debug(alice.net.unl.deconstruct())
    _log.debug(bob.net.unl.deconstruct())

    # alice.net.unl == bob.net.unl

    _log.debug(type(alice.net.unl))
    _log.debug(type(pyp2p.unl.UNL(value=bob.net.unl.value)))

    # exit()

    # Alice wants data from Bob.
    d = alice.data_request(
        "download",
        "ed980e5ef780d5b9ca1a6200a03302f2a91223044bc63dacc6d9f07eead663ab",
        0,
        bob.net.unl.value
    )

    def do_beep(ret):
        if ret != None:
            print("\a")

    d.addCallback(do_beep)

    """
    alice.data_request(
        "upload",
        "f2ca1bb6c7e907d06dafe4687e579fce76b37e4e93b7605022da52e6ccc26fd2",
        5,
        bob.net.unl.value
    )
    """

    # Main event loop.
    while 1:
        for client in [alice, bob]:
            if client == alice:
                _log.debug("Alice")
            else:
                _log.debug("Bob")
            process_transfers(client)

        time.sleep(0.002)


from typing import List

class Solution:
    def maxProfit(self, k: int, prices: List[int]) -> int:
        """
            https://leetcode.com/problems/best-time-to-buy-and-sell-stock-iv/
            Time Complexity - O(nk)
            'n' is the list of stock prices at a given day
            'k' is the no. of transactions
            Space Complexity - O(nk)
        """
        # edge prices
        if not prices:
            return 0

        # if the no of transactions is more than half the length of the list
        if k >= len(prices) // 2:
            return self._multiple_transactions(prices)

        # this takes for day 0 and k
        # profits[0, day] = 0; 0 transactions makes 0 profit
        # profits[transaction, 0] = 0; if there is only one price data point you can't make any transaction.
        # we cannot sell since we haven't bought any stock
        profits = [[0 for day in prices] for transaction in range(k + 1)]

        # t -> transaction
        # d-> day
        for t in range(1, k + 1):
            # store max between 0 and d
            max_thus_far = float('-inf')
            for d in range(1, len(prices)):
                # profit we had prev day with one less transaction  -  buying a stock on the prev day
                max_thus_far = max(max_thus_far, profits[t - 1][d - 1] - prices[d - 1])
                # Not selling anything or selling the stock with prev max,
                # (which includes the buy of the stock profit we made from the prev transaction)
                profits[t][d] = max(profits[t][d - 1], prices[d] + max_thus_far)
        return profits[-1][-1]

    def maxProfit(self, k: int, prices: List[int]) -> int:
        """
            https://leetcode.com/problems/best-time-to-buy-and-sell-stock-iv/
            Time Complexity - O(nk)
            'n' is the list of stock prices at a given day
            'k' is the no. of transactions
            Space Complexity - O(n)
        """
        # edge prices
        if not prices:
            return 0

        # if the no of transactions is more than half the length of the list
        if k >= len(prices) // 2:
            return self._multiple_transactions(prices)

        # even profits
        even_profits = [0 for d in prices]
        odd_profits = [0 for d in prices]
        for t in range(1, k + 1):
            max_thus_far = float('-inf')
            # even transaction
            if t % 2 == 0:
                current_profits = even_profits
                prev_profits = odd_profits
            else:  # odd transaction
                current_profits = odd_profits
                prev_profits = even_profits

            for d in range(1, len(prices)):
                max_thus_far = max(max_thus_far, prev_profits[d - 1] - prices[d - 1])
                current_profits[d] = max(current_profits[d - 1], prices[d] + max_thus_far)

        return even_profits[-1] if k % 2 == 0 else odd_profits[-1]

    def _multiple_transactions(self, prices: List[int]):
        i = profit = 0
        N = len(prices) - 1
        while i < N:
            # finding the last continuous day that the price is smallest
            while i < N and prices[i + 1] <= prices[i]:
                i += 1
            buy = prices[i]

            # finding the last continuous day that the price is biggest
            while i < N and prices[i + 1] > prices[i]:
                i += 1
            sell = prices[i]

            profit += sell - buy
        return profit

if __name__ == '__main__':
    print(Solution().maxProfit(2, [5, 11, 3, 50, 60, 90]))


import socket
import uuid
from http.server import HTTPServer
from typing import Any
from tornado import web
from tornado.ioloop import IOLoop
from tornado.routing import _RuleList

from ycyj_zhongtai.gateway.app.discovery import ZookeeperServiceInstance
from ycyj_zhongtai.libs.discovery.instance import ServiceInstance
from ycyj_zhongtai.libs.exception.base import CommonException, CommonErrorCode
from ycyj_zhongtai.libs.micro.registry import ZookeeperServiceRegistry
from ycyj_zhongtai.libs.zookeeper.client import ZookeeperMicroClient

class Application:

    def __init__(
            self,
            handlers: _RuleList = None,
            name: str = None,
            address: str = '0.0.0.0',
            port: int = 0,
            root_path: str = '/micro-service',
            **settings: Any
    ) -> None:
        self.handlers = handlers
        self.default_application = web.Application(self.handlers, settings=settings)
        self.service_registry = ZookeeperServiceRegistry(zookeeper=ZookeeperMicroClient(), root_path=root_path)
        self.service_name = name
        self.port = port
        self.address = address

    def start(self) -> None:

        if self.service_name is None:
            raise Exception()

        self.service_registry.start()
        self.service_registry.register_service(self._create_service_instance())
        self.listen()
        IOLoop.instance().start()

    def listen(self) -> HTTPServer:

        if self.port == 0:
            raise CommonException(CommonErrorCode.PortNoSetting_Error)

        if self.default_application is None:
            raise Exception()
        return self.default_application.listen(port=self.port, address=self.address)

    def _create_service_instance(self) -> 'ServiceInstance':

        instance_id = str(uuid.uuid4())
        service_id = self.service_name
        host = self._get_local_ip()
        port = self.port
        service = ZookeeperServiceInstance(instance_id=instance_id, service_id=service_id, host=host, port=port, metadata=dict())

        return service

    def _get_local_ip(self):
        s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
        s.connect(('8.8.8.8', 80))
        ip = s.getsockname()[0]
        s.close()
        return ip


"""
给定一个包含红色、白色和蓝色，一共 n 个元素的数组，原地对它们进行排序，使得相同颜色的元素相邻，并按照红色、白色、蓝色顺序排列。
此题中，我们使用整数 0、 1 和 2 分别表示红色、白色和蓝色。
注意:
不能使用代码库中的排序函数来解决这道题。
示例:
输入: [2,0,2,1,1,0]
输出: [0,0,1,1,2,2]

作者：力扣 (LeetCode)
链接：https://leetcode-cn.com/leetbook/read/all-about-array/x9wv2h/
来源：力扣（LeetCode）
著作权归作者所有。商业转载请联系作者获得授权，非商业转载请注明出处。
"""
from typing import List

class Solution:
    def sortColors(self, nums: List[int]) -> None:
        """
        Do not return anything, modify nums in-place instead.
        """
        n = len(nums)
        if n < 2:
            return
        for i in range(n):
            flag = False
            for j in range(n - 1 - i):
                if nums[j] > nums[j+1]:
                    tmp = nums[j]
                    nums[j] = nums[j+1]
                    nums[j+1] = tmp
                    flag = True
            if not flag:
                break

s = Solution()
test_n = [
    [2, 0, 2, 1, 1, 0],
]
for n in test_n:
    s.sortColors(n)
    print(n)


from django.contrib import admin
from .models import Tutorial, TutorialCategory, TutorialSeries
from tinymce.widgets import TinyMCE
from django.db import models

class TutorialAdmin(admin.ModelAdmin):
    # отображение полей при редактировании или добавлении
    # fields = ['tutorial_title', 'tutorial_content', 'tutorial_published']

    # Отображение полей секциями
    fieldsets = [
        ('Title/date/slug', {'fields': ['tutorial_title', 'tutorial_published', 'tutorial_slug', 'tutorial_series']}),
        ('Content', {'fields': ['tutorial_content']})
    ]

    formfield_overrides = {
        models.TextField: {'widget': TinyMCE()}
    }

admin.site.register(Tutorial, TutorialAdmin)
admin.site.register(TutorialCategory)
admin.site.register(TutorialSeries)


from django.contrib.auth.models import User
from django.db import models
from foodgram import settings

from users.models import CustomUser

class Tag(models.Model):
    name = models.CharField(
        verbose_name='Название ингредиента',
        blank=False,
        max_length=200,
        help_text='Укажите название ингредиента'
    )
    color = models.CharField(
        verbose_name=(u'Color'),
        max_length=7,
        help_text=(u'HEX color, as #RRGGBB'),
        )
    slug = models.SlugField(
        max_length=50,
        unique=True,
        verbose_name='Slug'
        )

    class Meta:
        verbose_name = 'Тег',
        verbose_name_plural = 'Теги'
        ordering = ['id']

class Ingredient(models.Model):
    name = models.CharField(
        verbose_name='Название ингредиента',
        blank=False,
        max_length=200,
        help_text='Укажите название ингредиента'
    )
    measurement_unit = models.CharField(
        verbose_name='Единица измерения',
        blank=False,
        max_length=200,
        help_text='Укажите единицу измерения'
    )

    class Meta:
        verbose_name = 'Игредиент',
        verbose_name_plural = 'Игредиенты'
        ordering = ['id']

class Recipe(models.Model):
    tags = models.ManyToManyField(
        Tag,
        blank=True,
        through='TagRecipe',
        related_name='recipes',
        verbose_name='Теги',
    )
    author = models.ForeignKey(
        CustomUser,
        blank=False,
        on_delete=models.CASCADE,
        related_name='recipes',
        verbose_name='Автор рецепта',
    )
    ingredients = models.ManyToManyField(
        Ingredient,
        blank=True,
        through='IngredientAmount',
        related_name='recipes',
        verbose_name='Ингредиенты',
    )
    name = models.CharField(
        verbose_name='Название',
        max_length=200,
        blank=False,
        help_text='Напишите название рецепта'
    )
    image = models.ImageField(
        upload_to='image/',
        null=False,
        verbose_name='Картинка рецепта',
    )
    text = models.TextField(
        verbose_name='Описание рецепта',
        blank=False,
        help_text='Добавьте сюда описание рецепта'
    )
    cooking_time = models.PositiveSmallIntegerField(
        verbose_name='Время приготовления в минутах',
        blank=False,
        help_text='Укажите Время приготовления в минутах',
    )

    class Meta:
        verbose_name = 'Рецепт',
        verbose_name_plural = 'Рецепты'
        ordering = ['id']

class IngredientAmount(models.Model):
    ingredient = models.ForeignKey(
        Ingredient,
        on_delete=models.CASCADE,
        blank=False,
        verbose_name='Ингредиент'
    )
    recipe = models.ForeignKey(
        Recipe,
        on_delete=models.CASCADE,
        blank=False,
        verbose_name='Рецепт'
    )
    amount = models.PositiveSmallIntegerField(
        blank=False,
        verbose_name='Количество игредиентов'
    )

    class Meta:
        verbose_name = 'Количество игредиентов',
        verbose_name_plural = 'Количества игредиентов'
        ordering = ['id']

class TagRecipe(models.Model):
    tag = models.ForeignKey(
        Tag,
        on_delete=models.CASCADE,
        blank=False,
        verbose_name='Тег'
    )
    recipe = models.ForeignKey(
        Recipe,
        on_delete=models.CASCADE,
        blank=False,
        verbose_name='Рецепт'
    )

    class Meta:
        verbose_name = 'Теги в рецепте',
        verbose_name_plural = 'Теги в рецептах'
        ordering = ['id']

class ShoppingCart(models.Model):
    owner = models.ForeignKey(
        CustomUser,
        on_delete=models.CASCADE,
        blank=False,
        verbose_name='Покупатель'
    )
    item = models.ForeignKey(
        Recipe,
        on_delete=models.CASCADE,
        blank=True,
        verbose_name='Товар'
        )

    class Meta:
        verbose_name = 'Список покупок',
        verbose_name_plural = 'Списки покупок'
        ordering = ['id']

class Favorite(models.Model):
    fav_user = models.ForeignKey(
        CustomUser,
        on_delete=models.CASCADE,
        blank=False,
        verbose_name='Пользователь'
    )
    fav_item = models.ForeignKey(
        Recipe,
        on_delete=models.CASCADE,
        blank=True,
        verbose_name='Рецепт в избранном'
        )

    class Meta:
        verbose_name = 'Избранное',
        verbose_name_plural = 'Избранные'
        ordering = ['id']

class Follow(models.Model):
    user = models.ForeignKey(
        settings.AUTH_USER_MODEL,
        on_delete=models.CASCADE,
        related_name="follower",
        verbose_name='Подпищик'
    )
    author = models.ForeignKey(
        settings.AUTH_USER_MODEL,
        on_delete=models.CASCADE,
        related_name="following",
        verbose_name='Автор'
    )

    class Meta:
        verbose_name = 'Подписка',
        verbose_name_plural = 'Подписки'
        ordering = ['id']


# train.csv, test.csv
# row_ jd: id of the check-in event
# xy: coordinates
# accuracy. location accuracy
# time: timestamp
# place. jd: id of the business, this is the target you are predicting，这个就是目标值
import pandas as pd
data = pd.read_csv()
#基本的数据处理
#缩小数据范围
data.query("x<2.5 & x>2 & y<1.5 & y>1.0")
#处理时间特征
time_value = pd.datetime(data["time"], unit = "s")
date = pd.DatetimeIndex(time_value)
date.weekday#可以得到星期天的数据
date.years#day,hour,

#将时间装进数据集
data["day"] = date.day

#过滤签到次数较少的地点
place_count = data.groupby("place_id").count()["row_id"]#按照palce_id分组,就拿到了一个签到次数的表,row_id表示出现的次数
place_count[place_count > 3]
databool = data["place_id"].isin(place_count[place_count > 3].index.values)#就得到了一组bool值，就是》3的布尔值
#最后对data进行bool索引就得到了bool值为真的数据
data_final = data[databool]

#筛选特征值和目标值
x = data_final[["x" , "y", "accuracy" , "day" , "weekday" , "hour"]]
y = data_final["palce_id"]

#数据集划分
from sklearn.model_selection import train_test_split
x_train, x_test, y_train, y_test = train_test_split(x,y)


import _pickle as pickle
import os
from multiprocessing import Process, Queue
import queue

import zmq
import torch
from torch.utils.tensorboard import SummaryWriter
import numpy as np
import time

import utils
from memory import BatchStorage
from wrapper import make_atari, wrap_atari_dqn
from model import DuelingDQN
from arguments import argparser
from datetime import datetime

def get_environ():
    actor_id = int(os.environ.get('ACTOR_ID', '-1'))
    n_actors = int(os.environ.get('N_ACTORS', '-1'))
    replay_ip = os.environ.get('REPLAY_IP', '-1')
    learner_ip = os.environ.get('LEARNER_IP', '-1')
    assert (actor_id != -1 and n_actors != -1)
    assert (replay_ip != '-1' and learner_ip != '-1')
    return actor_id, n_actors, replay_ip, learner_ip

def connect_param_socket(ctx, param_socket, learner_ip, actor_id):
    socket = ctx.socket(zmq.REQ)
    socket.connect("tcp://{}:52002".format(learner_ip))
    socket.send(pickle.dumps((actor_id, 1)))
    socket.recv()
    param_socket.connect('tcp://{}:52001'.format(learner_ip))
    socket.send(pickle.dumps((actor_id, 2)))
    socket.recv()
    print("Successfully connected to learner!")
    socket.close()

def recv_param(learner_ip, actor_id, param_queue):
    ctx = zmq.Context()
    param_socket = ctx.socket(zmq.SUB)
    param_socket.setsockopt(zmq.SUBSCRIBE, b'')
    param_socket.setsockopt(zmq.CONFLATE, 1)
    connect_param_socket(ctx, param_socket, learner_ip, actor_id)
    while True:
        data = param_socket.recv(copy=False)
        param = pickle.loads(data)
        param_queue.put(param)

def vector_exploration(args, actor_id, n_actors, replay_ip, param_queue):
    ctx = zmq.Context()
    batch_socket = ctx.socket(zmq.DEALER)
    batch_socket.setsockopt(zmq.IDENTITY, pickle.dumps('actor-{}'.format(actor_id)))
    batch_socket.connect('tcp://{}:51001'.format(replay_ip))
    outstanding = 0

    writer = SummaryWriter(comment="-{}-actor{}".format(args.env, actor_id))

    num_envs = args.num_envs_per_worker
    envs = [wrap_atari_dqn(make_atari(args.env), args) for _ in range(num_envs)]

    if args.seed is not None:
        seeds = args.seed + actor_id * num_envs + np.arange(num_envs)
        utils.set_global_seeds(seeds[0], use_torch=True)
        for seed, env in zip(seeds, envs):
            env.seed(int(seed))

    model = DuelingDQN(envs[0], args)
    model = torch.jit.trace(model, torch.zeros((1,4,84,84)))
    _actor_id = np.arange(num_envs) + actor_id * num_envs
    n_actors = n_actors * num_envs
    epsilons = args.eps_base ** (1 + _actor_id / (n_actors - 1) * args.eps_alpha)
    storages = [BatchStorage(args.n_steps, args.gamma) for _ in range(num_envs)]

    param = param_queue.get(block=True)
    model.load_state_dict(param)
    param = None
    print("%d: Received First Parameter!"%actor_id)

    actor_idx = 0
    tb_idx = 0
    episode_rewards = np.array([0] * num_envs)
    episode_lengths = np.array([0] * num_envs)
    states = np.array([env.reset() for env in envs])
    tb_dict = {key: [] for key in ['episode_reward', 'episode_length']}
    step_t = time.time()
    ref_t = 0
    sim_t = 0
    while True:
        if actor_idx * num_envs * n_actors <= args.initial_exploration_samples: # initial random exploration
            random_idx = np.arange(num_envs)
        else:
            random_idx, = np.where(np.random.random(num_envs) <= epsilons)
        _t = time.time()
        with torch.no_grad():
            states_tensor = torch.tensor(states, dtype=torch.float32)
            q_values = model(states_tensor).detach().numpy()
        ref_t += time.time() - _t
        actions = np.argmax(q_values, 1)
        actions[random_idx] = np.random.choice(envs[0].action_space.n, len(random_idx))

        for i, (state, q_value, action, env, storage) in enumerate(zip(states, q_values, actions, envs, storages)):
            _t = time.time()
            next_state, reward, done, _ = env.step(action)
            sim_t += time.time() - _t
            storage.add(np.array(state), reward, action, done, q_value, _t, episode_lengths[i])
            states[i] = next_state
            episode_rewards[i] += reward
            episode_lengths[i] += 1
            if done or episode_lengths[i] == args.max_episode_length:
                states[i] = env.reset()
                tb_idx += 1
                tb_dict["episode_reward"].append(episode_rewards[i])
                tb_dict["episode_length"].append(episode_lengths[i])
                episode_rewards[i] = 0
                episode_lengths[i] = 0
                if tb_idx % args.tb_interval == 0:
                    writer.add_scalar('actor/episode_reward_mean', np.mean(tb_dict['episode_reward']), tb_idx)
                    writer.add_scalar('actor/episode_reward_max', np.max(tb_dict['episode_reward']), tb_idx)
                    writer.add_scalar('actor/episode_reward_min', np.min(tb_dict['episode_reward']), tb_idx)
                    writer.add_scalar('actor/episode_reward_std', np.std(tb_dict['episode_reward']), tb_idx)
                    writer.add_scalar('actor/episode_length_mean', np.mean(tb_dict['episode_length']), tb_idx)
                    tb_dict['episode_reward'].clear()
                    tb_dict['episode_length'].clear()
                    writer.add_scalar('actor/step_time', (time.time() - step_t) / np.sum(episode_lengths), tb_idx)
                    writer.add_scalar('actor/step_inference_time', ref_t / np.sum(episode_lengths), tb_idx)
                    writer.add_scalar('actor/step_simulation_time', sim_t / np.sum(episode_lengths), tb_idx)
                    ref_t = 0
                    sim_t = 0
                    step_t = time.time()

        actor_idx += 1

        if actor_idx % args.update_interval == 0:
            try:
                param = param_queue.get(block=False)
                model.load_state_dict(param)
                print("%d: Updated Parameter.."%actor_id)
            except queue.Empty:
                pass

        if sum(len(storage) for storage in storages) >= args.send_interval * num_envs:
            batch = []
            prios = []
            for storage in storages:
                _batch, _prios = storage.make_batch()
                batch.append(_batch)
                prios.append(_prios)
                storage.reset()
            batch = [np.concatenate(v) for v in zip(*batch)]
            prios = np.concatenate(prios)
            data = pickle.dumps((batch, prios))
            batch, prios = None, None
            while outstanding >= args.max_outstanding:
                batch_socket.recv()
                outstanding -= 1
            batch_socket.send(data, copy=False)
            outstanding += 1
            # print(f"{actor_id}: send  batch {len(data)/1024} kb")

def main():
    actor_id, n_actors, replay_ip, learner_ip = get_environ()
    args = argparser()
    param_queue = Queue(maxsize=3)

    procs = [
        Process(target=vector_exploration, args=(args, actor_id, n_actors, replay_ip, param_queue)),
        Process(target=recv_param, args=(learner_ip, actor_id, param_queue)),
    ]

    for p in procs:
        p.start()
    for p in procs:
        p.join()
    return True

if __name__ == '__main__':
    os.environ["OMP_NUM_THREADS"] = "1"
    main()


import webbrowser, sys, pyperclip
'''This a simple and straight program that displays the power of the webbrowser module. To be able to run it from the command line, need to run it as a batchfile.
The conditions however makes it possible so that at all times either one of this variables contains something either with command line arguments or clipboard.'''

if len(sys.argv) > 1:
    open_myweb= ' '.join(sys.argv[1:])
elif len(sys.argv) == 1:
    open_myweb = pyperclip.paste()

webbrowser.open(open_myweb)


"""
    Create by: apenasrr
    Source: https://github.com/apenasrr/mass_videojoin
"""

import os
import pandas as pd
import datetime
import logging
from video_tools import change_width_height_mp4, get_video_details, \
                        join_mp4, split_mp4
from config_handler import handle_config_file
import unidecode
import natsort
import glob

def logging_config():

    logfilename = 'log-' + 'mass_videojoin' + '.txt'
    logging.basicConfig(filename=logfilename, level=logging.DEBUG,
                        format=' %(asctime)s-%(levelname)s-%(message)s')
    # set up logging to console
    console = logging.StreamHandler()
    console.setLevel(logging.INFO)
    # set a format which is simpler for console use
    formatter = logging.Formatter(' %(asctime)s-%(levelname)s-%(message)s')
    console.setFormatter(formatter)
    # add the handler to the root logger
    logging.getLogger('').addHandler(console)
    logger = logging.getLogger(__name__)

def clean_cmd():

    clear = lambda: os.system('cls')
    clear()

def df_sort_human(df):
    """
    Sort files and folders in human way.
    So after folder/file '1' comes '2', instead of '10' and '11'.
    Simple yet flexible natural sorting in Python.
    When you try to sort a list of strings that contain numbers,
    the normal python sort algorithm sorts lexicographically,
    so you might not get the results that you expect:
    More info: https://github.com/SethMMorton/natsort

    :input: DataFrame. With columns [file_folder, file_name]
    :return: DataFrame. Sort in a human way by [file_folder, file_name]
    """

    def sort_human(list_):

        list_ = natsort.natsorted(list_)

        return list_

    def sort_df_column_from_list(df, column_name, sorter):
        """
        :input: df: DataFrame
        :input: column_name: String
        :input: sorter: List
        :return: DataFrame
        """

        sorterIndex = dict(zip(sorter, range(len(sorter))))
        df['order'] = df[column_name].map(sorterIndex)
        df.sort_values(['order'], ascending = [True ], inplace = True)
        df.drop(['order', column_name], 1, inplace = True)

        return df

    column_name = 'path_file'
    df[column_name] = df['file_folder'] + '\\' + df['file_name']
    list_path_file = df[column_name].tolist()
    sorter = sort_human(list_path_file)
    df = sort_df_column_from_list(df, column_name, sorter)

    return df

def gen_report(path_dir):

    # TODO input more file video extension:
    ## https://dotwhat.net/type/video-movie-files

    l=[]
    for root, dirs, files in os.walk(path_dir):
        for file in files:
            file_lower = file.lower()
            if file_lower.endswith((".mp4", ".avi", ".webm", '.ts', '.vob',
                                    '.mov')):
                print(file)

                path_file = os.path.join(root, file)
                dict_inf = get_video_details(path_file)
                (mode, ino, dev, nlink, uid,
                gid, size, atime, mtime, ctime) = os.stat(path_file)
                mtime = datetime.datetime.fromtimestamp(mtime)
                d={}
                d['mtime']=mtime
                d['file_folder'] = root
                d['file_name'] = file
                d['file_size'] = os.path.getsize(path_file)

                try:
                    d['duration'] = dict_inf['duration']
                except:
                    logging.error('video without duration: {path_file}')
                    d['duration'] = ''

                d['bitrate'] = dict_inf['bitrate']
                d['video_codec'] = dict_inf['video']['codec']
                d['video_profile'] = dict_inf['video']['profile']
                d['video_resolution'] = dict_inf['video']['resolution']
                d['video_bitrate'] = dict_inf['video']['bitrate']

                # some videos dont have audio
                try:
                    d['audio_codec'] = dict_inf['audio']['codec']
                    d['audio_frequency'] = dict_inf['audio']['frequency']
                    d['audio_bitrate'] = dict_inf['audio']['bitrate']
                except:
                    d['audio_codec'] = ''
                    d['audio_frequency'] = ''
                    d['audio_bitrate'] = ''
                d['video_resolution_to_change'] = ''
                l.append(d)
    df = pd.DataFrame(l)

    return df

def get_video_details_with_group(df):

    df['key_join_checker'] = df['audio_codec'] + '-' + \
                             df['video_codec'] + '-' + \
                             df['video_resolution']

    # set group_encode
    df['group_encode'] = 1
    for index, row in df.iterrows():
        if index>0:
            group_encode_value_prev = df.loc[index-1, 'group_encode']
            if row['key_join_checker'] != df.loc[index-1, 'key_join_checker']:
                df.loc[index, 'group_encode'] = group_encode_value_prev + 1
            else:
                df.loc[index, 'group_encode'] = group_encode_value_prev

    return df

def get_list_chunk_videos_from_group(df, group_no, max_size_mb):

    max_size_bytes = max_size_mb * 1024**2
    mask = df['group_encode'].isin([group_no])

    df['file_path'] = df['file_folder'] + '\\' + \
                      df['file_name']

    df_group = df.loc[mask, :]
    list_chunk_videos = []
    chunk_size = 0
    list_videos = []
    for index, row in df_group.iterrows():
        if chunk_size + row['file_size'] > max_size_bytes:
            logging.info(f'join video from {len(list_videos)} files')
            list_chunk_videos.append(list_videos)

            list_videos = []
            chunk_size = 0

        list_videos.append(row['file_path'])
        chunk_size += row['file_size']

    if len(list_videos) > 0:
        logging.info(f'join video from {len(list_videos)} files')
        list_chunk_videos.append(list_videos)
        list_videos = []

    logging.info(f'group {group_no} will generate ' + \
                 f'{len(list_chunk_videos)} videos')
    return list_chunk_videos

def get_list_chunk_videos(df, max_size_mb):

    list_group = df['group_encode'].unique().tolist()
    list_final = []

    for group_no in list_group:
        group_no = str(group_no)
        list_chunk_videos = get_list_chunk_videos_from_group(df, group_no,
                                                             max_size_mb)
        list_final += list_chunk_videos
        print('')

    return list_final

def get_name_dir_origin():

    name_file_folder_name = get_txt_folder_origin()
    dir_name_saved = get_txt_content(name_file_folder_name)

    return dir_name_saved

def get_path_folder_output_video():

    path_folder_output = get_name_dir_origin()
    folder_name = 'output_' + path_folder_output
    ensure_folder_existence([folder_name])

    path_folder_output_video = os.path.join(folder_name, 'output_videos')
    ensure_folder_existence([path_folder_output_video])

    return path_folder_output_video

def join_videos(df, max_size_mb, start_index_output):

    # path_folder_output = userpref_folderoutput()
    path_folder_output = get_path_folder_output_video()

    # default_filename_output = input('Enter a default name for the joined ' +\
                                    # 'videos: ')
    default_filename_output = get_name_dir_origin()

    df['file_path'] = df['file_folder'] + '\\' + df['file_name']
    list_chunk_videos = get_list_chunk_videos(df, max_size_mb)
    df['file_output'] = ''

    for index, list_file_path in enumerate(list_chunk_videos):
        file_count = index + start_index_output
        file_name_output = f'{default_filename_output}-%03d.mp4' % file_count
        file_path_output = os.path.join(path_folder_output, file_name_output)
        join_mp4(list_file_path, file_path_output)

        # register file_output in dataframe
        mask_files_joined = df['file_path'].isin(list_file_path)
        df.loc[mask_files_joined, 'file_output'] = file_name_output

    print(f'total: {len(list_chunk_videos)} videos')

    return df

def exclude_all_files_from_folder(path_folder):

        path_folder_regex = os.path.join(path_folder, '*')
        r = glob.glob(path_folder_regex)
        for i in r:
            os.remove(i)

def make_reencode(df):

    folder_script_path = get_folder_script_path()
    path_folder_encoded = os.path.join(folder_script_path, 'videos_encoded')
    exclude_all_files_from_folder(path_folder_encoded)

    # exclude_folder_videos_encoded()

    df['file_folder_origin'] = df['file_folder']
    df['file_name_origin'] = df['file_name']
    df['file_size_origin'] = df['file_size']
    df['video_resolution_origin'] = df['video_resolution']
    mask_df_to_reencode = ~df['video_resolution_to_change'].isna()
    df_to_reencode = df.loc[mask_df_to_reencode, :]

    folder_script_path = get_folder_script_path()
    path_folder_dest = os.path.join(folder_script_path, 'videos_encoded')
    for index, row in df_to_reencode.iterrows():
        size_width, size_height = row['video_resolution_to_change'].split('x')

        path_file_origin = os.path.join(row['file_folder_origin'],
                                        row['file_name_origin'])
        print(path_file_origin)
        # path_folder_dest = r'videos_encoded'
        path_file_name_dest = str(index) + '.mp4'
        path_file_dest = os.path.join(path_folder_dest,
                                      path_file_name_dest)

        # todo reencode
        # input path_folder_dest in column file_folder
        df.loc[index, 'file_folder'] = os.path.abspath(path_folder_dest)
        # input path_file_name_dest in column file_name
        df.loc[index, 'file_name'] = path_file_name_dest

        change_width_height_mp4(path_file_origin, size_height,
                                size_width, path_file_dest)

        file_size = os.stat(path_file_dest).st_size
        df.loc[index, 'file_size'] = file_size
        df.loc[index, 'video_resolution'] = row['video_resolution_to_change']

        # from encoded video get video metadata
        metadata = get_video_details(path_file_dest)
        # register video metadata
        df.loc[index, 'bitrate'] = metadata['bitrate']
        df.loc[index, 'video_bitrate'] = metadata['video']['bitrate']
        df.loc[index, 'video_codec'] = metadata['video']['codec']
        df.loc[index, 'audio_codec'] = metadata['audio']['codec']
        df.loc[index, 'audio_bitrate']  = metadata['audio']['bitrate']
        df.loc[index, 'duration'] = metadata['duration']

    return df

def menu_ask():

    # ptbr
    # print('1-Gerar planilha listando os arquivos')
    # print('2-Processar reencode dos vídeos marcados na coluna '+
          # '"video_resolution_to_change"')
    # print('3-Agrupar vídeos em grupos de até 1 gb com mesmo codec e resolução')

    # eng
    print('1-Generate worksheet listing the files')
    print('2-Process reencode of videos marked in column ' +
          '"video_resolution_to_change"')
    print('3-Group videos into groups up to 1 gb with the same codec ' + \
          'and resolution')

    # ptbr
    # msg_type_answer = 'Digite sua resposta: '

    # eng
    msg_type_answer = 'Type your answer: '
    make_report = int(input(f'\n{msg_type_answer}'))
    if make_report == 1:
        return 1
    elif make_report == 2:
        return 2
    elif make_report == 3:
        return 3
    else:
        # ptbr
        # msg_invalid_option = "Opção não disponível"

        # eng
        msg_invalid_option = "Invalid option"
        raise MyValidationError(msg_invalid_option)

def df_insert_row(row_number, df, row_value):
    """
    A customized function to insert a row at any given position in the
     dataframe.
    source: https://www.geeksforgeeks.org/insert-row-at-given-position-in-pandas-dataframe/
    :input: row_number: Int.
    :input: df: Dataframe.
    :input: row_value: Int.
    :return: Dataframe. df_result |
             Boolean. False. If the row_number was invalid.
    """

    if row_number > df.index.max()+1:
        print("df_insert_row: Invalid row_number")
        return False

    # Slice the upper half of the dataframe
    df1 = df[0:row_number]

    # Store the result of lower half of the dataframe
    df2 = df[row_number:]

    # Inser the row in the upper half dataframe
    df1.loc[row_number]=row_value

    # Concat the two dataframes
    df_result = pd.concat([df1, df2])

    # Reassign the index labels
    df_result.index = [*range(df_result.shape[0])]

    # Return the updated dataframe
    return df_result

def search_to_split_videos(df, mb_limit):

    def preprocess_df_split(df):

        df['file_path'] = df['file_folder'] + '\\' + df['file_name']
        df['split_file_folder_origin'] = ''
        df['split_file_name_origin'] = ''
        df['split_file_size_origin'] = ''

        return df

    def get_dict_row_dest(dict_row_origin, pathfile_output):

        path_folder_dest = os.path.split(pathfile_output)[0]
        file_name_dest = os.path.split(pathfile_output)[1]
        file_size_dest = os.stat(pathfile_output).st_size

        dict_row_dest = dict_row_origin.copy()
        dict_row_dest['split_file_folder_origin'] = \
            dict_row_origin['file_folder']
        dict_row_dest['split_file_name_origin'] = dict_row_origin['file_name']
        dict_row_dest['split_file_size_origin'] = dict_row_origin['file_size']

        dict_row_dest['file_folder'] = path_folder_dest
        dict_row_dest['file_name'] = file_name_dest
        dict_row_dest['file_size'] = file_size_dest

        return dict_row_dest

    def get_row_number_from_filepath(df, file_path_origin):

        path_folder_origin = os.path.split(file_path_origin)[0]
        file_name_origin = os.path.split(file_path_origin)[1]

        mask1 = df['file_folder'].isin([path_folder_origin])
        mask2 = df['file_name'].isin([file_name_origin])
        mask_file = mask1 & mask2

        df_row_origin = df.loc[mask_file, :]
        dict_row_origin = df_row_origin.iloc[0]
        row_number = df_row_origin.index.values[0]

        return row_number

    def delete_fileorigin(df, file_path_origin):

        row_number = get_row_number_from_filepath(df, file_path_origin)
        df = df.drop(df.index[[row_number]])
        df = df.reset_index(drop=True)
        return df

    def update_df_files(df, file_path_origin, list_filepath_output):

        def include_rows_new_files(df, filepath_output):

            pathfile_output = os.path.abspath(filepath_output)
            # find the row_number of origin file
            row_number = get_row_number_from_filepath(df, file_path_origin)
            dict_row_origin = df.loc[row_number,:]
            dict_row_dest = get_dict_row_dest(dict_row_origin, pathfile_output)
            df = df_insert_row(row_number=row_number, df=df,
                                   row_value=dict_row_dest)
            return df

        for filepath_output in list_filepath_output:
            # include the rows corresponding to the new files created
            df = include_rows_new_files(df=df,
                                        filepath_output=filepath_output)

        # delete the file origin row
        df = delete_fileorigin(df=df, file_path_origin=file_path_origin)

        return df

    def get_list_file_path_origin(df, size_limit):

        mask_to_be_split = df['file_size'] > size_limit
        df_to_be_split = df.loc[mask_to_be_split, :]
        list_file_path_origin = df_to_be_split.loc[:, 'file_path'].tolist()

        return list_file_path_origin

    df = preprocess_df_split(df)

    recoil_sec = 10
    # TODO estimate the recoil_mbsize by video bitrate
    recoil_mbsize = 10
    size_limit = (mb_limit-recoil_mbsize) * 1024**2
    list_file_path_origin = get_list_file_path_origin(df=df,
                                                      size_limit=size_limit)
    folder_script_path = get_folder_script_path()
    output_folder_path = os.path.join(folder_script_path, 'videos_splitted')
    exclude_all_files_from_folder(output_folder_path)

    for file_path_origin in list_file_path_origin:
        list_filepath_output = split_mp4(largefile_path=file_path_origin,
                                         recoil=recoil_sec,
                                         mb_limit=mb_limit,
                                         output_folder_path=output_folder_path)

        df = update_df_files(df=df, file_path_origin=file_path_origin,
                             list_filepath_output=list_filepath_output)

    return df

def userpref_folderoutput():

    path_file = os.path.join('config', 'config.txt')
    variable_name = 'path_folder_output'
    variable_value = \
        handle_config_file(path_file, variable_name,
                           set_value=None, parse=True)
    path_folder_output = variable_value['path_folder_output'][0]

    print(f'Use the folder path output as {path_folder_output}?')
    answer_use = input('(None for yes) Answer: ')
    if answer_use == '':
        pass
    else:
        path_folder_output = input('Inform the folder path output: ')
        handle_config_file(path_file, variable_name,
                           set_value=path_folder_output, parse=False)

    return path_folder_output

def userpref_size_per_file_mb():

    folder_script_path = get_folder_script_path()
    path_file = os.path.join(folder_script_path, 'config', 'config.txt')
    variable_name = 'size_per_file_mb'

    question = 'What should be the maximum size of each ' + \
               'file in mb (e.g.: 500)? '

    variable_value = \
        handle_config_file(path_file, variable_name,
                           set_value=None, parse=True)
    value_got = variable_value[variable_name][0]

    print(f'The maximum size of each file will be {value_got}. Ok?')
    answer_use = input('(None for yes) Answer: ')
    if answer_use == '':
        pass
    else:
        value_got = input(question)
        handle_config_file(path_file, variable_name,
                           set_value=value_got, parse=False)

    return value_got

def ensure_folder_existence(folders_path):
    """
    :input: folders_path: List
    """

    for folder_path in folders_path:
        existence = os.path.isdir(folder_path)
        if existence is False:
            os.mkdir(folder_path)

def ensure_folders_existence():

    folder_script_path_relative = os.path.dirname(__file__)
    folder_script_path = os.path.realpath(folder_script_path_relative)

    folders_name = ['ts', 'videos_encoded', 'config', 'videos_splitted']
    folders_path = []
    for folder_name in folders_name:
        folder_path = os.path.join(folder_script_path, folder_name)
        folders_path.append(folder_path)

    ensure_folder_existence(folders_path)

def get_txt_content(file_path):

    file = open(file_path,'r', encoding='utf-8')
    file_content = file.readlines()
    file_content = ''.join(file_content)
    file.close()

    return file_content

def create_txt(file_path, stringa):

    f = open(file_path, "w", encoding='utf8')
    f.write(stringa)
    f.close()

def get_folder_name_normalized(path_dir):

    def normalize_string_to_link(string_actual):

        string_new = unidecode.unidecode(string_actual)

        for c in r"!@#$%^&*()[]{};:,./<>?\|`~-=_+":
            string_new = string_new.translate({ord(c): "_"})

        string_new = string_new.replace(' ', '_')
        string_new = string_new.replace('___', '_')
        string_new = string_new.replace('__', '_')

        return string_new

    dir_name = os.path.basename(path_dir)
    dir_name_normalize = normalize_string_to_link(dir_name)

    return dir_name_normalize

def prefill_video_resolution_to_change(df):

    df['key_join_checker'] = df['audio_codec'] + '-' + \
                             df['video_codec'] + '-' + \
                             df['video_resolution']

    df_key = df[['key_join_checker', 'duration', 'video_resolution']].copy()
    df_key['duration_timedelta'] = pd.to_timedelta(df_key['duration'])
    df_key['duration_min'] = df_key['duration_timedelta'].dt.total_seconds()/60

    df_key.drop('duration', axis=1, inplace=True)

    df_key_agg = df_key.groupby(['key_join_checker',
                                 'video_resolution'])['duration_min'].agg('sum')
    # convert in dataframe
    df_key_agg = df_key_agg.reset_index()
    # sort dataframe
    df_key_agg = df_key_agg.sort_values(['duration_min'], ascending=[False])

    print('\n', df_key_agg)
    index_max = df_key_agg['duration_min'].idxmax()
    key_join_main = df_key_agg.loc[index_max, 'key_join_checker']
    video_resolution_main = df_key_agg.loc[index_max, 'video_resolution']

    # informar quantidade de minutos para reencodar
    mask_to_convert = ~df_key_agg.index.isin([index_max])
    minutes_to_reencode = df_key_agg.loc[mask_to_convert, 'duration_min'].sum()
    minutes_total = df_key_agg['duration_min'].sum()
    percent_to_reencode = minutes_to_reencode/minutes_total
    print(f'The main profile is "{key_join_main}"')

    # informar percentual de minutos para reencodar
    # the command ':.1f' fix 1 digit after decimal point
    print(f'There is {minutes_to_reencode:.1f} minutes ' + \
          f'({percent_to_reencode*100:.0f}%) to reencode')

    mask_resolution_to_change = ~df['key_join_checker'].isin([key_join_main])
    df.loc[mask_resolution_to_change,
           'video_resolution_to_change'] = video_resolution_main
    df.drop('key_join_checker', axis=1, inplace=True)

    return df

def save_upload_folder_name(path_dir, file_folder_name):

    dir_name_normalize = get_folder_name_normalized(path_dir)
    create_txt(file_path=file_folder_name, stringa=dir_name_normalize)

def create_report_backup(df, path_file_report, tag):

    path_folder = os.path.dirname(path_file_report)
    file_name = os.path.basename(path_file_report)
    file_name_without_extension = os.path.splitext(file_name)[0]
    file_name_backup = file_name_without_extension + "_" + tag + ".xlsx"
    path_file_backup = os.path.join(path_folder, file_name_backup)
    df.to_excel(path_file_backup, index=False)

def step_create_report_filled(path_dir, path_file_report):

    df = gen_report(path_dir)
    # sort path_file by natural human way
    df = df_sort_human(df)
    # prefill column video_resolution_to_change
    df = prefill_video_resolution_to_change(df)
    df.to_excel(path_file_report, index=False)

    #Make backup
    create_report_backup(df=df, path_file_report=path_file_report, tag='origin')

def set_make_reencode(path_file_report):

    df = pd.read_excel(path_file_report)
    df = make_reencode(df)
    df.to_excel(path_file_report, index=False)

    #make backup
    create_report_backup(df=df, path_file_report=path_file_report, tag='reencode')
    print('\nReencode finished')

def set_group_column(path_file_report):

    # update video_details with groups

    df = pd.read_excel(path_file_report)
    df = get_video_details_with_group(df)
    df.to_excel(path_file_report, index=False)
    print(f"File '{path_file_report}' was updated with " + \
          f"group column to fast join\n")
    # backup is not performed here as the grouping can be adjusted manually

def set_split_videos(path_file_report, mb_limit):

    df = pd.read_excel(path_file_report)

    #backup group, after adjusted manually
    create_report_backup(df=df, path_file_report=path_file_report, tag='grouped')

    # Find for file_video too big and split them
    df = search_to_split_videos(df, mb_limit=mb_limit)

    df.to_excel(path_file_report, index=False)

    create_report_backup(df=df, path_file_report=path_file_report, tag='splited')

def set_join_videos(path_file_report, mb_limit, start_index_output):

    df = pd.read_excel(path_file_report)
    df = join_videos(df, mb_limit, start_index_output)
    df.to_excel(path_file_report, index=False)

    #backup joined
    create_report_backup(df=df, path_file_report=path_file_report, tag='joined')

def get_folder_script_path():

    folder_script_path_relative = os.path.dirname(__file__)
    folder_script_path = os.path.realpath(folder_script_path_relative)

    return folder_script_path

def set_path_file_report():

    folder_path_output_relative = 'output_' + get_name_dir_origin()
    ensure_folder_existence([folder_path_output_relative])
    path_file_report = os.path.join(folder_path_output_relative,
                                    'video_details.xlsx')

    return path_file_report

def get_txt_folder_origin():

    file_folder_name = 'folder_files_origin.txt'

    return file_folder_name

def get_start_index_output():

    print('Start output file count with what value?')
    add_num = input('(None for 1) Answer: ')
    if add_num == '':
        add_num = 1
    else:
        add_num = int(add_num)
    return add_num

def main():

    ensure_folders_existence()
    file_name_folder_origin = get_txt_folder_origin()

    path_file_report = set_path_file_report()

    menu_answer = menu_ask()

    if menu_answer == 1:
        # create Dataframe of video details
        path_dir = input('\nPaste the folder link where are the video files: ')

        # save in txt, the folder name
        save_upload_folder_name(path_dir, file_name_folder_origin)

        path_file_report = set_path_file_report()

        step_create_report_filled(path_dir, path_file_report)

        print(f'\nIf necessary, change the reencode plan in the column ' + \
              f'"video_resolution_to_change"')

        break_point = input('Type Enter to continue')
        clean_cmd()
        main()
        return

    elif menu_answer == 2:

        # reencode videos mark in column video_resolution_to_change
        set_make_reencode(path_file_report)

        break_point = input('Review the file and then type something to ' + \
                            'continue.')
        clean_cmd()
        main()
        return

    elif menu_answer == 3:

        mb_limit = int(userpref_size_per_file_mb())

        # establishes separation criteria for the join videos step
        set_group_column(path_file_report)

        break_point = input('Review the file and then type something to ' + \
                            'start the process that look for videos that ' + \
                            'are too big and should be splitted')

        set_split_videos(path_file_report, mb_limit)

        # set start index output file
        start_index_output = get_start_index_output()

        # join all videos
        set_join_videos(path_file_report, mb_limit, start_index_output)
        return
    else:

        return

if __name__ == "__main__":
    logging_config()
    main()


import numpy as np
import networkx as nx
import matplotlib.pyplot as plt
import matplotlib.colors as colors
import matplotlib.cm as cmx

"""
Plotting code used for general visualization
"""

# tie_decay_result = np.load("tie_decay_1e3.npy").item()

print ("Now start plotting")

G = nx.Graph()
tie_decay_edges = [(k[0], k[1], v) for k, v in tie_decay_result.items()
                  if v >= 10 ** -100]
G.add_weighted_edges_from(tie_decay_edges)

edges,weights = zip(*nx.get_edge_attributes(G,'weight').items())
colors = ['r' if w > 10 ** -10 else 'b' for w in weights]
width = [1 if w > 10 ** -10 else 0.1 for w in weights]
pos = nx.spring_layout(G)

plt.figure()
nx.draw(G, pos, node_size=5, node_color='b', edgelist=edges, edge_color=colors, width=width)
plt.savefig("tie_decay_example.png")
plt.close()


import numpy as np
import matplotlib.pyplot as plt
from scipy.optimize import curve_fit
import math
import scipy.stats
from mpl_toolkits.mplot3d import Axes3D
from multiprocessing import Pool
import random
import glob
import os

cpath = "/afs/.ir.stanford.edu/users/a/r/arider/gas_sim_data/s_100mum"

def build_arr(fs):
    arr = np.array([])
    for f in fs:
        arr = np.append(arr, np.loadtxt(f))
    return arr

def get_data(path):
    xfs = glob.glob(os.path.join(path, "*x.txt"))
    yfs = glob.glob(os.path.join(path, "*y.txt"))
    zfs = glob.glob(os.path.join(path, "*z.txt"))
    return build_arr(xfs),  build_arr(yfs),  build_arr(zfs)

class Fstats:
    def __init__(self, xa, ya, za, n):
        self.fs = np.array([xa, ya, za])
        self.n = n
        self.nsim = len(xa)

def gauss(x, *p):
    #a gaussian function for fitting to distributuions
    A, mu, sigma = p
    return A*np.exp(-(x-mu)**2/(2.*sigma**2))

def fit_fdist(fsobj, axis = 0, npb = 10., p0 = [100.,0., 0.00002]):
    #plots histogram of average force during simulation.
    nbins = int(len(fsobj.fs[axis, :])/npb)
    h, be = np.histogram(fsobj.fs[axis, :], nbins)
    bc = (be[1:] + be[:-1])/2.
    popt, pcov = curve_fit(gauss, bc, h, p0 = p0)
    mustr = '{:.2e}'.format(popt[1])
    sigstr = '{:.2e}'.format(np.sqrt(pcov[1, 1]))
    pltx = np.linspace(np.min(bc), np.max(bc), 100)
    plt.plot(pltx, gauss(pltx, *popt), label = \
             'mean ' + mustr+" $\pm$ " +sigstr, linewidth = 2.)
    plt.hist(fsobj.fs[axis, :], nbins)
    plt.xlabel('Force after %i collisions'%fsobj.nsim)
    plt.ylabel('Frequency in %i simulations'%fsobj.n)
    plt.legend()
    plt.show()
    return pcov

def plter(xarr, yarr, zarr):
    #plots the trajectories
    fig = plt.figure()
    ax = fig.add_subplot(111, projection='3d')
    ax.plot(xarr, yarr, zarr, '.-')
    ax.set_xlabel('x')
    ax.set_ylabel('y')
    ax.set_zlabel('z')
    #ax.set_xlim([-1, 1])
    #ax.set_ylim([-1, 1])
    #ax.set_zlim([-1, 1])
    plt.show()

xarr, yarr, zarr = get_data(cpath)
fsobj = Fstats(xarr, yarr, zarr, 10000000)
fit_fdist(fsobj, axis = 1)

#plter(xarr, yarr, zarr)


# GOAL OF THIS SOFTWARE: print scraped data in Terminal
## please install PYTHON3, CHROMEDRIVER, SELENIUM, then proceed
### then enter in Terminal ```pip install -r setup.py```
#### NOW, you can run this script in Terminal with ```python3 scrape_bundles.py```
##### ♲ reuse & recycle & have a nice day ♲

# headless browser
from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.common.exceptions import TimeoutException
# raw file data tool
import requests

# make sure you also add CHROMEDRIVER to system path (I enclosed the program in this folder so your device might find it ;)
driver = webdriver.Chrome('./chromedriver')
# read here if your device has problems w CHROMEDRIVER PATH stuff:
# https://github.com/SeleniumHQ/selenium/wiki/ChromeDriver

# surfs Incognito => scrapes better ^_^
option = webdriver.ChromeOptions()
option.add_argument("--incognito")

# launches Chrome
browser = webdriver.Chrome(executable_path='./chromedriver', options=option)

# grab text of any
 any level inside of class="mw-category-group"
# 'div[@class="mw-category-group"]//h3'
# // any h3 in a div with class 'mw-category-group'

# surfs to this URL
page_this_time = 'Farm_buildings'
url_this_time = f"https://stardewvalleywiki.com/Category:{page_this_time}"
print(f"Let's scrape this site: {url_this_time}")
browser.get(url_this_time)

###
##
# internet lag timeout (error handling)
timeout = 30
try:
	# checking for element on page, to test connection (the only
 with id='pt-createaccount' )
	WebDriverWait(browser, timeout).until(EC.visibility_of_element_located((By.XPATH, "//li[@id='pt-createaccount']")))
except TimeoutException:
	print("This page takes too long to load-- try again?")
	browser.quit()
#
##
###
# so we are scraping the URL above {url_this_time} for info...
# confirm the next bit yourself with Inspect in your browser, then modify this script as YOU need w this guide:
# https://gist.github.com/LeCoupa/8c305ec8c713aad07b14
# ENJOY CODING
#
# the URL target has info we need: this XPATH selector id's only the HTML we want
# it's always listed (are

s) inside
 elements w this ⬇ class prop, and they are all ⬇  elements
results = browser.find_elements_by_xpath("//div[@class='mw-category-group']/ul/li/a")
# now let's collect the exact prop from these HTML elements
result_s = [x.get_attribute('title') for x in results]
# and print them all to Terminal
for result_s in result_s:
    print(result_s)
print(f"This concludes the scraping of: {url_this_time}")
# ENJOY CODING
##### ♲ reuse & recycle & have a nice day ♲


import pymongo
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.cluster import KMeans
from sklearn.metrics import adjusted_rand_score

def getText(tweet_collection):
    usernames,hashtags,text = [],[],[]
    alldata = tweet_collection.find()
    for tweet in alldata:

        ulower = str(tweet['username']).lower()
        htlower = str(tweet['hashtags']).lower()
        tlower = str(tweet['text']).lower()
        usernames.append(ulower)
        hashtags.append(htlower)
        text.append(tlower)

    return usernames,hashtags,text

def cluster(usernames,hashtags,texts):
    vectorizer = TfidfVectorizer(stop_words='english')
    n = 12

    username_vector = vectorizer.fit_transform(usernames)
    hashtag_vector = vectorizer.fit_transform(hashtags)
    text_vector = vectorizer.fit_transform(texts)

    usernames_cluster = KMeans(n_clusters=n,init='k-means++',max_iter=100,n_init=1)
    usernames_cluster.fit(username_vector)

    hashtags_cluster = KMeans(n_clusters=n,init='k-means++',max_iter=100,n_init=1)
    hashtags_cluster.fit(hashtag_vector)

    text_cluster = KMeans(n_clusters=n,init='k-means++',max_iter=100,n_init=1)
    text_cluster.fit(text_vector)

    return vectorizer,usernames_cluster,hashtags_cluster,text_cluster

def topResult(vectorizer,group):
    n = 12
    centroids = group.cluster_centers_.argsort()[:,::-1]
    text = vectorizer.get_feature_names()
    for centre in range(n):
        print ("CLUSTER %d:" % centre)
        for i in centroids[centre, :5]:
            print (' %s' % text[i])

if __name__ == "__main__":
    MONGO_HOST = 'mongodb://localhost/twitterdb'
    client = pymongo.MongoClient(MONGO_HOST)
    db = client.twitterdb
    tweet_collection = db.tweets

    sc,ht,t = getText(tweet_collection)
    vec,scK,htK,tK = cluster(sc,ht,t)

    print("++++ USERNAME CLUSTERS ++++")
    topResult(vec,scK)
    print("++++ HASHTAG CLUSTERS ++++")
    topResult(vec,htK)
    print("++++ TEXT CLUSTERS ++++")
    topResult(vec,tK)


# coding: utf-8

from django.shortcuts import render
from django.shortcuts import redirect
from django.http import HttpResponse, Http404
from django.template import loader
#from django.core.exceptions import ObjectDoesNotExist

from .forms import *
from .models import *
from django.contrib.auth.models import*
from django.contrib.auth import *
from bs4 import BeautifulSoup
from django.template.response import *
from datetime import *
import bs4
import re
import pickle
import os
from nltk import *
from bs4 import BeautifulSoup
from random import *
from .creation_graph import *
from unidecode import unidecode
from django.utils.safestring import mark_safe

app_name = 'revendication'

#_____________________vue_______________________#

def page_accueil(request):
	if request.user.is_authenticated:
		pass
	else:
		return render(request, 'revendications/varthing.html')
	from django.contrib.auth.models import User
	utilisateur = request.user

	def creer_les_evenements_du_calendriers ():

		evenements = Evenement.objects.all()
		fichier_evenement = u""
		for evenement in evenements:
			ligne =  evenement.titre + "/" + str(evenement.date) + "/" + str(evenement.id) + "/" + str(evenement.description) +"ggg"
			fichier_evenement = fichier_evenement + ligne

		fichier_evenement = unidecode(fichier_evenement).encode("utf-8")
		#print ("************************** fichier_evenement", fichier_evenement)
		return fichier_evenement

	def revendications():
		propositions = Proposition.objects.all()
		mes_propositions = Proposition.objects.filter(soutien__user = utilisateur, soutien__lien = "SO")
		date_M1 = datetime.now()-timedelta(30)
		liste= []
        #progression?
		for p in propositions:
			soutiens_M1 = Soutien.objects.filter(date__lte = date_M1, propositions = p)
			nb_soutiens = len(soutiens_M1)
			p.progression = nb_soutiens
			liste.append(p)
			if p in mes_propositions:
				p.mienne = "oui"
			else:
				p.mienne = "non"
		#print("liste______", liste)
		return liste

	def documents():
		documents = Document.objects.all()
		return documents

	def evenements():
		liste_de_mes_evenements = Evenement.objects.filter(soutien__user = utilisateur)
		liste_de_mes_propositions = Proposition.objects.filter(soutien__user = utilisateur)
		evenements = Evenement.objects.all()
		liste = []
		for ev in evenements:
			if ev in liste_de_mes_evenements:
				ev.mienne = "oui"
			else:
				ev.mienne = "non"
			liste.append(ev)
		return liste

	def competences():
		liste_de_mes_competences = Competence.objects.filter(soutien__user = utilisateur)
		liste_de_mes_propositions = Proposition.objects.filter(soutien__user = utilisateur)
		competences = Competence.objects.all()
		liste = []
		for comp in competences:
			if comp in liste_de_mes_competences :
				comp.mienne = "oui"
			else:
				comp.mienne = "non"
			liste.append(comp)
		return liste

	def organisations():
	#creer la liste des organisations dont je soutiens les propositions
		liste_de_mes_organisations = Organisation.objects.filter(soutien__user = utilisateur)
		liste_de_mes_propositions = Proposition.objects.filter(soutien__user = utilisateur)
		organisations = Organisation.objects.all()
		liste = []
		for org in organisations:
			if org in liste_de_mes_organisations:
				org.mienne = "oui"
			else:
				org.mienne = "non"
			liste.append(org)
		return liste

	def petitions():
		liste_de_mes_petitions = Petition.objects.filter(soutien__user = utilisateur)
		liste_de_mes_propositions = Proposition.objects.filter(soutien__user = utilisateur)
		petitions = Petition.objects.all()
		liste = []
		for p in petitions:
			if p in liste_de_mes_petitions :
				p.mienne = "oui"
			else:
				p.mienne = "non"
			liste.append(p)
		return liste

	class formulaire:
		def __init__ (self):
			self.revendication = RevendicationForm
			self.petition =PetitionForm
			self.evenement= EvenementForm

	def trier_les_elements_par_dates(liste):
		liste = sorted(liste, key=lambda x: x[1])
		liste.reverse()

	def creer_les_datas(utilisateur):

		class Datas:
			def __init__ (self):
				self.evenements = evenements()
				self.competences = competences()
				self.documents = documents()
				self.petitions = petitions()
				self.revendications = revendications()
				self.organisations = organisations()
				self.suggestions = self.revendications

				#formulaires:
				class formulaire:
					def __init__ (self):
						self.revendication = RevendicationForm
						self.petition =PetitionForm
						self.evenement= EvenementForm
				self.formulaires = formulaire()

		datas = Datas()
		return datas

	datas = creer_les_datas(utilisateur)

	try:
		onglet = request.session["onglet"]
	except:
		onglet = "vide"

	graph_a =  graph_accueil()

	request.session['message']="vide"
	request.session["proposition_id"]="toutes"

	message = request.session['message']
	return render(request, 'revendications/page_accueil.html', {"datas":datas, "graph_accueil":mark_safe(graph_a),"onglet": onglet, "message":mark_safe(message)})