"""
This is a prototyp of the functionality that a red cross field worker would have
"""
import sys
import time
import asyncio
import json
import pprint
import ast
from invite_new_user import invite_new_user
from tkinter import *
from tkinter import ttk
from tkinter import simpledialog

#MAIN PARAMETERS
from indy import pool, ledger, wallet, did, crypto, anoncreds, pairwise
from indy.error import IndyError

#MAIN PARAMETERS
from settings import *

global pool_handle

async def accept_sovrin_invite():
    try:

        print_log('\n2. Open ledger and get handle\n')
        pool_handle = await get_pool_handle()

        print_log('Start accepting sovrin invite')
        name = 'Alice'
        # new_user Agent
        print_log('Create wallet config ')
        wallet_config_new_user = json.dumps({"id": name +"_wallet"})
        wallet_credentials_new_user = json.dumps({"key": name + "_wallet_key"})

        print_log('Create the new wallet of the affected_person (this will be done on their own device) ')
        try:
            await wallet.create_wallet(wallet_config_new_user, wallet_credentials_new_user)
        except IndyError:
            print('after error')
            await wallet.delete_wallet(wallet_config_new_user, wallet_credentials_new_user)
            await wallet.create_wallet(wallet_config_new_user, wallet_credentials_new_user)

        new_user_wallet = await wallet.open_wallet(wallet_config_new_user, wallet_credentials_new_user)

        # new_user Agent
        (new_user_creator_did, new_user_creator_key) = await did.create_and_store_my_did(new_user_wallet, "{}")
        print_log('new_user creator DID: ', new_user_creator_did)
        print_log('new_user creator key: ', new_user_creator_key)

        print_log("Open the connections request")
        with open('connection_request.json') as f:
            connection_request = json.load(f)

        # new_user Agent
        connection_response = json.dumps({
            'did': new_user_creator_did,
            'verkey': new_user_creator_key,
            'nonce': connection_request['nonce']
        })

        # new_user Agent
        creator_new_user_verkey = await did.key_for_did(pool_handle, new_user_wallet, connection_request['did'])
        print_log('creator new_user verkey: ', creator_new_user_verkey)

        print_log('Anoncryps a connection response and write it to a file (can only be openen by the redcorss issuer)')
        print_log('IRL this should be send through a server or something')
        # new_user Agent
        anoncrypted_connection_response = await crypto.anon_crypt(creator_new_user_verkey, connection_response.encode('utf-8'))
        #print_log('Anoncrypted_connection_response : ', anoncrypted_connection_response)

        f = open('anoncrypted_connection_response', 'wb')
        f.write(anoncrypted_connection_response)
        f.close()

        creator_did_info_json = json.dumps({
            'did': connection_request['did'],
            'verkey': creator_new_user_verkey
        })
        print_log('Store their did')
        #store_their_did_return = await did.store_their_did(new_user_wallet,  creator_did_info_json)
        print_log('Create Pairwise')
        await pairwise.create_pairwise(new_user_wallet, connection_request['did'], new_user_creator_did, 'random_metadata')

        await wallet.close_wallet(new_user_wallet)
        print_log('DONE DONE DONE DONE DONE DONE DONE DONE DONE DONE DONE DONE DONE DONE DONE')

    except IndyError as e:
        print('Error occurred: %s' % e)

async def new_user_creates_did():
    try:
        pool_handle = await get_pool_handle()
        print_log('AP creates a did for the connection with the RC')
        new_user_wallet = await open_ap_wallet('Alice')
        (new_user_did, new_user_key) = await did.create_and_store_my_did(new_user_wallet, "{}")
        print_log('new_user DID: ', new_user_did)
        print_log('new_user key: ', new_user_key)

        with open('connection_request.json') as f:
            connection_request = json.load(f)

        print_log('new_user creates message with DID and verkey')
        # new_user Agent
        new_user_did_info_json = json.dumps({
            'did': new_user_did,
            'verkey': new_user_key
        })

        #print_log("store_their_did_return", store_their_did_return)
        print_log('new_user key: ', new_user_key)

        print_log('new_user authenticates and encrypts the message by calling auth_crypt')

        creator_new_user_verkey = await did.key_for_did(pool_handle, new_user_wallet, connection_request['did'])
        print_log('get pairwise')
        new_user_creator_did =  await pairwise.get_pairwise(new_user_wallet, connection_request['did'])
        print_log(" new_user_creator_did", new_user_creator_did)
        new_user_creator_did = ast.literal_eval(new_user_creator_did)
        new_user_creator_key = await did.key_for_did(pool_handle, new_user_wallet, new_user_creator_did['my_did'])
        # new_user Agent
        print(new_user_wallet, new_user_creator_key, creator_new_user_verkey)
        print_log('auth_crypt')
        authcrypted_new_user_did_info_json = \
            await crypto.auth_crypt(new_user_wallet, new_user_creator_key, creator_new_user_verkey, new_user_did_info_json.encode('utf-8'))

        print_log('auth_crypt a its user information (did and verkey) and write it to a file (can only be openen by the redcorss issuer)')
        print_log('IRL this should be send through a server or something')
        f = open('authcrypted_new_user_did_info_json', 'wb')
        f.write(authcrypted_new_user_did_info_json)
        f.close()

        await wallet.close_wallet(new_user_wallet)

        print_log('DONE DONE DONE DONE DONE DONE DONE DONE DONE DONE DONE DONE DONE DONE DONE')
    except IndyError as e:
        print('Error occurred: %s' % e)

async def respond_to_cred_offer():
    try:
        # Start of alice
        pool_handle = await get_pool_handle()
        new_user_wallet = await open_ap_wallet('Alice')

        print_log('Open Auth crypted cred offer from the rc id issuer')
        file = open("authcrypted_identity_cred_offer_json", "rb")
        authcrypted_identity_cred_offer_json = file.read()
        file.close()

        print_log("I'm assuming we know their did as we're getting message from them, im cheating here by opening this json to get it")
        with open('connection_request.json') as f:
            connection_request = json.load(f)
        rc_ap_did = connection_request['did']

        rc_verkey, identity_cred_offer_json = await auth_decrypt_using_pairwise_storage(pool_handle, new_user_wallet, rc_ap_did, authcrypted_identity_cred_offer_json)

        print_log("identity_cred_offer_json", str(identity_cred_offer_json))

        ap_rc_did = await get_pairwise_string(new_user_wallet, rc_ap_did)
        print_log('\n PA reads schema & cred def from ledger\n')
        # Alice Agent
        get_schema_request = await ledger.build_get_schema_request(ap_rc_did, identity_cred_offer_json['schema_id'])
        print_log("submit_request")
        get_schema_response = await ledger.submit_request(pool_handle, get_schema_request)
        identity_schema = await ledger.parse_get_schema_response(get_schema_response)

        print_log("identity_schema", identity_schema)

        get_cred_def_request = await ledger.build_get_cred_def_request(ap_rc_did, identity_cred_offer_json['cred_def_id'])
        print_log("Na get_cred_def_request")
        get_cred_def_response = await ledger.submit_request(pool_handle, get_cred_def_request)
        print_log("Na get_cred_def_response ")
        (redcross_identity_cred_def_id, redcross_identity_cred_def_json) = await ledger.parse_get_cred_def_response(get_cred_def_response)
        print_log('redcross_identity_cred_def_json', redcross_identity_cred_def_json)

        print_log('\n Alice creates master secret (NO IDEA WHY)\n')
        # Alice Agent

        ap_master_secret_id = await anoncreds.prover_create_master_secret(new_user_wallet, None)
        print_log('ap_master_secret_id: ',ap_master_secret_id)

        print_log('\n Alice creates Identity-credential request\n')

        # Alice Agent
        identity_cred_offer_json = json.dumps(identity_cred_offer_json)
        (identity_cred_request_json, identity_cred_request_metadata_json) = \
            await anoncreds.prover_create_credential_req(new_user_wallet, ap_rc_did, identity_cred_offer_json,
                                                         redcross_identity_cred_def_json, ap_master_secret_id)
        print_log('identity_cred_request_json: ',identity_cred_request_json)
        print_log('identity_cred_request_metadata_json: ',identity_cred_request_metadata_json)

        print_log("Store items on AP device")
        with open('identity_cred_request_metadata_json.json', 'w') as fp:
            json.dump(identity_cred_request_metadata_json, fp)
        with open('redcross_identity_cred_def_json.json', 'w') as fp:
            json.dump(redcross_identity_cred_def_json, fp)

        print_log('\n ###### MAGICALLY SEND identity_cred_request_json FROM ap TO redCross_idIssuer\n')
        print_log('\n In this case we auth encrypt it and store it\n')

        authcrypted_identity_cred_request_json = \
            await auth_crypt_using_pairwise_storage(pool_handle, new_user_wallet, rc_ap_did, rc_verkey, identity_cred_request_json)

        f = open('authcrypted_identity_cred_request_json', 'wb')
        f.write(authcrypted_identity_cred_request_json)
        f.close()

        print_log('\n ###### INTERFACE\n')
        await wallet.close_wallet(new_user_wallet)
        print_log('DONE DONE DONE DONE DONE DONE DONE DONE DONE DONE DONE DONE DONE DONE DONE DONE DONE DONE DONE DONE DONE DONE DONE DONE ')
    except IndyError as e:
        print('Error occurred: %s' % e)

async def AP_store_credential():
    try:
        pool_handle = await get_pool_handle()
        new_user_wallet = await open_ap_wallet('Alice')
        print_log('\n ###### MAGICALLY RECEIVED  identity_cred_json FROM redCross_idIssuer\n')
        file = open("authcrypted_identity_cred_json", "rb")
        authcrypted_identity_cred_json = file.read()
        file.close()

        print_log("I'm assuming we know rc did as we're getting message from them, im cheating here by opening this json to get it")
        with open('connection_request.json') as f:
            connection_request = json.load(f)
        rc_ap_did = connection_request['did']

        print_log('\n Alice decrypts identity credential in wallet\n')

        rc_verkey, identity_cred_json = await auth_decrypt_using_pairwise_storage(pool_handle, new_user_wallet, rc_ap_did, authcrypted_identity_cred_json)

        identity_cred_json = json.dumps(identity_cred_json)

        print_log("Open cred def and req metadata from ap device")
        with open('identity_cred_request_metadata_json.json') as f:
            identity_cred_request_metadata_json = json.load(f)
        with open('redcross_identity_cred_def_json.json') as f:
            redcross_identity_cred_def_json = json.load(f)

        print_log('\n AP stores identity credential in wallet\n')
        # Alice Agent
        await anoncreds.prover_store_credential(new_user_wallet, None, identity_cred_request_metadata_json,
                                                  identity_cred_json, redcross_identity_cred_def_json, None)

        print_log('DONE DONE DONE DONE DONE DONE DONE DONE DONE DONE DONE DONE DONE DONE DONE DONE DONE DONE DONE DONE DONE DONE DONE DONE ')
    except IndyError as e:
        print('Error occurred: %s' % e)

def run_accept_sovrin_invite():
    print_log("The script is starting")
    asyncio.set_event_loop(asyncio.new_event_loop())
    loop = asyncio.get_event_loop()
    loop.run_until_complete(accept_sovrin_invite())
    loop.close()

def run_new_user_creates_did():
    print_log("The script is starting")
    asyncio.set_event_loop(asyncio.new_event_loop())
    loop = asyncio.get_event_loop()
    loop.run_until_complete(new_user_creates_did())
    loop.close()

def run_respond_to_cred_offer():
    print_log("The script is starting")
    asyncio.set_event_loop(asyncio.new_event_loop())
    loop = asyncio.get_event_loop()
    loop.run_until_complete(respond_to_cred_offer())
    loop.close()

def run_AP_store_credential():
    print_log("The script is starting")
    asyncio.set_event_loop(asyncio.new_event_loop())
    loop = asyncio.get_event_loop()
    loop.run_until_complete(AP_store_credential())
    loop.close()

def select_part(input):
    input = int(input)
    if input ==1:
        run_accept_sovrin_invite()
    elif input ==2:
        run_new_user_creates_did()
    elif input ==3:
        run_respond_to_cred_offer()
    elif input ==4:
        run_AP_store_credential()
    else:
        print_log("Please enter a valid input")

print_log("Person Affected interface:")
# Pharsing args one by one
total = len(sys.argv)
if total ==1:
    print_log("Please enter the number of which part you want to run")
    print_log("1. Accept the sovrin invite")
    print_log("2. Create did to communicate with RC")
    print_log("3. Respond to cred offer")
    print_log("4. Store red cross credential")
    input = input("Enter number: ")
    select_part(input)
elif total == 2:
    input = sys.argv[1]
    select_part(input)
else:
    print_log("Please do not enter more than 1 arg")


from django import forms
from django.forms import ModelForm
from django.utils.translation import ugettext_lazy as _
from django.contrib.auth import get_user_model

from .models import Invitation
from .adapters import get_invitations_adapter
from .exceptions import AlreadyInvited, AlreadyAccepted, UserRegisteredEmail

class CleanEmailMixin(object):

    def validate_invitation(self, email):
        if Invitation.objects.all_valid().filter(
                email__iexact=email, accepted=False):
            raise AlreadyInvited
        elif Invitation.objects.filter(
                email__iexact=email, accepted=True):
            raise AlreadyAccepted
        elif get_user_model().objects.filter(email__iexact=email):
            raise UserRegisteredEmail
        else:
            return True

    def clean_email(self):
        email = self.cleaned_data["email"]
        email = get_invitations_adapter().clean_email(email)

        errors = {
            "already_invited": _("This e-mail address has already been"
                                 " invited."),
            "already_accepted": _("This e-mail address has already"
                                  " accepted an invite."),
            "email_in_use": _("An active user is using this e-mail address"),
        }
        try:
            self.validate_invitation(email)
        except(AlreadyInvited):
            raise forms.ValidationError(errors["already_invited"])
        except(AlreadyAccepted):
            raise forms.ValidationError(errors["already_accepted"])
        except(UserRegisteredEmail):
            raise forms.ValidationError(errors["email_in_use"])
        return email

class InviteForm(ModelForm):
    class Meta:
        model = Invitation
        fields = ['email', 'group']

class InvitationAdminAddForm(forms.ModelForm, CleanEmailMixin):
    email = forms.EmailField(
        label=_("E-mail"),
        required=True,
        widget=forms.TextInput(attrs={"type": "email", "size": "30"}))
    group = forms.CharField(label="Group")

    def save(self, *args, **kwargs):
        cleaned_data = super(InvitationAdminAddForm, self).clean()
        email = cleaned_data.get("email")
        params = {'email': email}
        if cleaned_data.get("inviter"):
            params['inviter'] = cleaned_data.get("inviter")
        instance = Invitation.create(**params)
        instance.send_invitation(self.request)
        super(InvitationAdminAddForm, self).save(*args, **kwargs)
        return instance

    class Meta:
        model = Invitation
        fields = ("email", "inviter")

class InvitationAdminChangeForm(forms.ModelForm):

    class Meta:
        model = Invitation
        fields = '__all__'


import pytest
import numpy as np
import pandas as pd
from inference.inference import Infer

data = pd.DataFrame()

class TestInfer:

    def test_from_two_arrays(self):
        test_infer = Infer(data)
        ar1 = np.random.rand(1, 10)
        ar2 = np.random.rand(1, 10)
        df = test_infer.from_two_arrays(ar1, ar2)
        value_in_df = "value" in df
        id_in_df = "group_id" in df
        assert value_in_df is True, id_in_df is True

    def test_from_two_proportions(self):
        test_infer = Infer(data)
        prop_1 = 75
        prop_2 = 25
        len_data1 = len_data2 = 10
        df = test_infer.from_two_proportions(prop_1, prop_2, len_data1, len_data2)
        value_in_df = "value" in df
        id_in_df = "group_id" in df
        assert value_in_df is True, id_in_df is True

    def test_test_independence(self):
        pass


"""
Tools for fitting data using non-linear least squares, and other approaches.
"""
import copy
import numpy as np
import scipy
import scipy.optimize

def fit_model(img, model_fn, init_params, fixed_params=None, sd=None, bounds=None, model_jacobian=None, **kwargs):
    """
    Fit 2D model function to an image. Any Nan values in the image will be ignored. Wrapper for fit_least_squares

    :param np.array img: nd array
    :param model_fn: function f(p)
    :param list[float] init_params: p = [p1, p2, ..., pn]
    :param list[boolean] fixed_params: list of boolean values, same size as init_params. If None, no parameters will be fixed.
    :param sd: uncertainty in parameters y. e.g. if experimental curves come from averages then should be the standard
    deviation of the mean. If None, then will use a value of 1 for all points. As long as these values are all the same
    they will not affect the optimization results, although they will affect chi squared.
    :param tuple[tuple[float]] bounds: (lbs, ubs). If None, -/+ infinity used for all parameters.
    :param model_jacobian: Jacobian of the model function as a list, [df/dp[0], df/dp[1], ...]. If None, no jacobian used.
    :param kwargs: additional key word arguments will be passed through to scipy.optimize.least_squares
    :return:
    """
    to_use = np.logical_not(np.isnan(img))

    # if all sd's are nan or zero, set to 1
    if sd is None or np.all(np.isnan(sd)) or np.all(sd == 0):
        sd = np.ones(img.shape)

    # handle uncertainties that will cause fitting to fail
    if np.any(sd == 0) or np.any(np.isnan(sd)):
        sd[sd == 0] = np.nanmean(sd[sd != 0])
        sd[np.isnan(sd)] = np.nanmean(sd[sd != 0])

    # function to be optimized
    def err_fn(p): return np.divide(model_fn(p)[to_use].ravel() - img[to_use].ravel(), sd[to_use].ravel())

    # if it was passed, use model jacobian
    if model_jacobian is not None:
        def jac_fn(p): return [v[to_use] / sd[to_use] for v in model_jacobian(p)]
    else:
        jac_fn = None

    results = fit_least_squares(err_fn, init_params, fixed_params=fixed_params, bounds=bounds,
                                model_jacobian=jac_fn, **kwargs)

    return results

def fit_least_squares(model_fn, init_params, fixed_params=None, bounds=None, model_jacobian=None, **kwargs):
    """
    Wrapper for non-linear least squares fit function scipy.optimize.least_squares which handles fixing parameters
    and calculating fit uncertainty.

    :param model_fn: function of model parameters p which returns an array, where the sum of squares of this array is
    minimized. e.g. if we have a set of data points x_i and we make measurements y_i with uncertainties sigma_i, and we have
    a model m(p, x_i)
     then f(p) = [(m(p, x_i) - y_i) / sigma_i]
    :param list[float] init_params: p = [p1, p2, ..., pn]
    :param list[boolean] fixed_params: list of boolean values, same size as init_params. If None, no parameters will be fixed.
    :param tuple[tuple[float]] bounds: (lbs, ubs). If None, -/+ infinity used for all parameters.
    :param model_jacobian: Jacobian of the model function as a list, [df/dp[0], df/dp[1], ...]. If None, no jacobian used.
    :param kwargs: additional key word arguments will be passed through to scipy.optimize.least_squares
    :return:
    """

    # get default fixed parameters
    if fixed_params is None:
        fixed_params = [False for _ in init_params]

    # default bounds
    if bounds is None:
        bounds = (tuple([-np.inf] * len(init_params)), tuple([np.inf] * len(init_params)))

    init_params = np.array(init_params, copy=True)
    # ensure initial parameters within bounds, if not fixed
    for ii in range(len(init_params)):
        if (init_params[ii] < bounds[0][ii] or init_params[ii] > bounds[1][ii]) and not fixed_params[ii]:
            raise ValueError("Initial parameter at index %d had value %0.2g, which was outside of bounds (%0.2g, %0.2g"
                             % (ii, init_params[ii], bounds[0][ii], bounds[1][ii]))

    if np.any(np.isnan(init_params)):
        raise ValueError("init_params cannot include nans")

    if np.any(np.isnan(bounds)):
        raise ValueError("bounds cannot include nans")

    # if some parameters are fixed, we need to hide them from the fit function to produce correct covariance, etc.
    # Idea: map the "reduced" (i.e. not fixed) parameters onto the full parameter list.
    # do this by looking at each parameter. If it is supposed to be "fixed" substitute the initial parameter. If not,
    # then get the next value from pfree. We find the right index of pfree by summing the number of previously unfixed parameters
    free_inds = [int(np.sum(np.logical_not(fixed_params[:ii]))) for ii in range(len(fixed_params))]

    def pfree2pfull(pfree):
        return np.array([pfree[free_inds[ii]] if not fp else init_params[ii] for ii, fp in enumerate(fixed_params)])

    # map full parameters to reduced set
    def pfull2pfree(pfull): return np.array([p for p, fp in zip(pfull, fixed_params) if not fp])

    # function to minimize the sum of squares of, now as a function of only the free parameters
    def err_fn_pfree(pfree): return model_fn(pfree2pfull(pfree))

    if model_jacobian is not None:
        def jac_fn_free(pfree): return pfull2pfree(model_jacobian(pfree2pfull(pfree))).transpose()
    init_params_free = pfull2pfree(init_params)
    bounds_free = (tuple(pfull2pfree(bounds[0])), tuple(pfull2pfree(bounds[1])))

    # non-linear least squares fit
    if model_jacobian is None:
        fit_info = scipy.optimize.least_squares(err_fn_pfree, init_params_free, bounds=bounds_free, **kwargs)
    else:
        fit_info = scipy.optimize.least_squares(err_fn_pfree, init_params_free, bounds=bounds_free,
                                                jac=jac_fn_free, x_scale='jac', **kwargs)
    pfit = pfree2pfull(fit_info['x'])

    # calculate chi squared
    nfree_params = np.sum(np.logical_not(fixed_params))
    red_chi_sq = np.sum(np.square(model_fn(pfit))) / (model_fn(init_params).size - nfree_params)

    # calculate covariances
    try:
        jacobian = fit_info['jac']
        cov_free = red_chi_sq * np.linalg.inv(jacobian.transpose().dot(jacobian))
    except np.linalg.LinAlgError:
        cov_free = np.nan * np.zeros((jacobian.shape[1], jacobian.shape[1]))

    cov = np.nan * np.zeros((len(init_params), len(init_params)))
    ii_free = 0
    for ii, fpi in enumerate(fixed_params):
        jj_free = 0
        for jj, fpj in enumerate(fixed_params):
            if not fpi and not fpj:
                cov[ii, jj] = cov_free[ii_free, jj_free]
                jj_free += 1
                if jj_free == nfree_params:
                    ii_free += 1

    result = {'fit_params': pfit, 'chi_squared': red_chi_sq, 'covariance': cov,
              'init_params': init_params, 'fixed_params': fixed_params, 'bounds': bounds,
              'cost': fit_info['cost'], 'optimality': fit_info['optimality'],
              'nfev': fit_info['nfev'], 'njev': fit_info['njev'], 'status': fit_info['status'],
              'success': fit_info['success'], 'message': fit_info['message']}

    return result

def get_moments(img, order=1, coords=None, dims=None):
    """
    Calculate moments of distribution of arbitrary size
    :param img:
    :param order: order of moments to be calculated
    :param coords: list of coordinate arrays for each dimension e.g. [y, x], where y, x etc. are 1D arrays
    :param dims: dimensions to be summed over. For example, given roi_size 3D array of size Nz x Ny x Nz, calculate the 2D
    moments of each slice by setting dims = [1, 2]
    :return:
    """

    # todo: does not compute any cross moments, e.g. X*Y

    if dims is None:
        dims = range(img.ndim)

    if coords is None:
        coords = [np.arange(s) for ii, s in enumerate(img.shape) if ii in dims]
    # ensure coords are float arrays to avoid overflow issues
    coords = [np.array(c, dtype=np.float) for c in coords]

    if len(dims) != len(coords):
        raise ValueError('dims and coordinates must have the same length')

    # weight summing only over certain dimensions
    w = np.nansum(img, axis=tuple(dims), dtype=np.float)

    # as trick to avoid having to meshgrid any of the coordinates, we can use NumPy's array broadcasting. Because this
    # looks at the trailing array dimensions, we need to swap our desired axis to be the last dimension, multiply by the
    # coordinates to do the broadcasting, and then swap back
    moments = [np.nansum(np.swapaxes(np.swapaxes(img, ii, img.ndim-1) * c**order, ii, img.ndim-1),
               axis=tuple(dims), dtype=np.float) / w
               for ii, c in zip(dims, coords)]

    return moments

# fit data to gaussians
def fit_gauss1d(y, init_params=None, fixed_params=None, sd=None, x=None, bounds=None):
    """
    Fit 1D Gaussian

    :param y:
    :param init_params: [A, cx, sx, bg]
    :param fixed_params:
    :param sd:
    :param x:
    :param bounds:
    :return:
    """

    # get coordinates if not provided
    if x is None:
        x = np.arange(len(y))

    # get default initial parameters
    if init_params is None:
        init_params = [None] * 4
    else:
        init_params = copy.deepcopy(init_params)

    # guess reasonable parameters if not provided
    if np.any([ip is None for ip in init_params]):
        to_use = np.logical_not(np.isnan(y))

        bg = np.nanmean(y.ravel())
        A = np.max(y[to_use].ravel()) - bg

        cx, = get_moments(y, order=1, coords=[x])
        m2x, = get_moments(y, order=2, coords=[x])
        sx = np.sqrt(m2x - cx ** 2)

        ip_default = [A, cx, sx, bg]

        # set any parameters that were None to the default values
        for ii in range(len(init_params)):
            if init_params[ii] is None:
                init_params[ii] = ip_default[ii]

    if bounds is None:
        bounds = ((-np.inf, x.min(), 0, -np.inf),
                  (np.inf, x.max(), x.max() - x.min(), np.inf))

    fn = lambda p: gauss2d(x, np.zeros(x.shape), [p[0], p[1], 0, p[2], 1, p[3], 0])
    jacob_fn = lambda p: gauss2d_jacobian(x, np.zeros(x.shape), [p[0], p[1], 0, p[2], 1, p[3], 0])

    result = fit_model(y, fn, init_params, fixed_params=fixed_params,
                       sd=sd, bounds=bounds, model_jacobian=jacob_fn)

    pfit = result['fit_params']
    fit_fn = lambda x: gauss2d(x, np.zeros(x.shape), [pfit[0], pfit[1], 0, pfit[2], 1, pfit[3], 0])

    return result, fit_fn

def fit_gauss2d(img, init_params=None, fixed_params=None, sd=None, xx=None, yy=None, bounds=None):
    """
    Fit 2D gaussian function. The angle theta is defined clockwise from the x- (or y-) axis. NOTE: be careful
    with this when looking at results using e.g. matplotlib.imshow, as this will display the negative y-axis on top.

    :param img: 2D image to fit
    :param init_params: [A, cx, cy, sx, sy, bg, theta]
    :param fixed_params: list of boolean values, same size as init_params.
    :param sd: uncertainty in parameters y. e.g. if experimental curves come from averages then should be the standard
    deviation of the mean
    :param xx: 2D array, same size as image (use this instead of 1D array because want to preserve ability to fit on
    non-regularly spaced grids, etc.)
    :param yy:
    :param bounds: (lbs, ubs)
    :return dict results:
    :return fit_fn:
    """

    # get coordinates if not provided
    if xx is None or yy is None:
        xx, yy = np.meshgrid(range(img.shape[1]), range(img.shape[0]))

    # get default initial parameters
    if init_params is None:
        init_params = [None] * 7
    else:
        init_params = copy.deepcopy(init_params)

    # guess reasonable parameters if not provided
    if np.any([ip is None for ip in init_params]):
        to_use = np.logical_not(np.isnan(img))

        bg = np.nanmean(img.ravel())
        A = np.max(img[to_use].ravel()) - bg

        cy, cx = get_moments(img, order=1, coords=[yy[:, 0], xx[0, :]])
        m2y, m2x = get_moments(img, order=2, coords=[yy[:, 0], xx[0, :]])
        with np.errstate(invalid='ignore'):
            sx = np.sqrt(m2x - cx ** 2)
            sy = np.sqrt(m2y - cy ** 2)

        ip_default = [A, cx, cy, sx, sy, bg, 0]

        # set any parameters that were None to the default values
        for ii in range(len(init_params)):
            if init_params[ii] is None:
                init_params[ii] = ip_default[ii]

    # replace any bounds which are none with default guesses
    lbs_default = (-np.inf, xx.min(), yy.min(), 0, 0, -np.inf, -np.inf)
    ubs_default = (np.inf, xx.max(), yy.max(), xx.max() - xx.min(), yy.max() - yy.min(), np.inf, np.inf)

    if bounds is None:
        bounds = (lbs_default, ubs_default)
    else:
        lbs = tuple([b if b is not None else lbs_default[ii] for ii, b in enumerate(bounds[0])])
        ubs = tuple([b if b is not None else ubs_default[ii] for ii, b in enumerate(bounds[1])])
        bounds = (lbs, ubs)

    # do fitting
    result = fit_model(img, lambda p: gauss2d(xx, yy, p), init_params, fixed_params=fixed_params,
                       sd=sd, bounds=bounds, model_jacobian=lambda p: gauss2d_jacobian(xx, yy, p))

    # model function
    def fit_fn(x, y): return gauss2d(x, y, result['fit_params'])

    return result, fit_fn

def fit_sum_gauss2d(img, ngaussians, init_params, fixed_params=None, sd=None, xx=None, yy=None, bounds=None):
    """
    Fit 2D gaussian function. The angle theta is defined clockwise from the x- (or y-) axis. NOTE: be careful
    with this when looking at results using e.g. matplotlib.imshow, as this will display the negative y-axis on top.

    :param img: 2D image to fit
    :param init_params: [A1, cx1, cy1, sx1, sy1, theta1, A2, cx2, ..., thetan, bg]
    :param fixed_params: list of boolean values, same size as init_params.
    :param sd: uncertainty in parameters y. e.g. if experimental curves come from averages then should be the standard
    deviation of the mean
    :param xx: 2D array, same size as image (use this instead of 1D array because want to preserve ability to fit on
    non-regularly spaced grids, etc.)
    :param yy:
    :param bounds: (lbs, ubs)
    :return:
    """

    # get coordinates if not provided
    if xx is None or yy is None:
        xx, yy = np.meshgrid(range(img.shape[1]), range(img.shape[0]))

    nparams = 6 * ngaussians + 1
    # get default initial parameters
    if init_params is None:
        init_params = [None] * nparams
    else:
        init_params = copy.deepcopy(init_params)

    if bounds is None:
        bounds = [[-np.inf, xx.min(), yy.min(), 0, 0, -np.inf] * ngaussians + [-np.inf],
                  [ np.inf, xx.max(), yy.max(), xx.max() - xx.min(), yy.max() - yy.min(), np.inf] * ngaussians + [np.inf]]

    result = fit_model(img, lambda p: sum_gauss2d(xx, yy, p), init_params, fixed_params=fixed_params,
                       sd=sd, bounds=bounds, model_jacobian=lambda p: sum_gauss2d_jacobian(xx, yy, p))

    pfit = result['fit_params']

    def fn(x, y):
        return sum_gauss2d(x, y, pfit)

    return result, fn

def fit_half_gauss1d(y, init_params=None, fixed_params=None, sd=None, x=None, bounds=None):
    """
    Fit function that has two Gaussian halves with different sigmas and offsets but match smoothly at cx

    :param y:
    :param init_params: [A1, cx, sx1, bg1, sx2, bg2]
    :param fixed_params:
    :param sd:
    :param x:
    :param bounds:
    :return:
    """

    # get coordinates if not provided
    if x is None:
        x = np.arange(len(y))

    # get default initial parameters
    if init_params is None:
        init_params = [None] * 6
    else:
        # init_params = copy.deepcopy(init_params)
        init_params = [p for p in init_params]

    # guess reasonable parameters if not provided
    if np.any([ip is None for ip in init_params]):
        to_use = np.logical_not(np.isnan(y))

        bg = np.nanmean(y.ravel())
        A = np.max(y[to_use].ravel()) - bg

        cx, = get_moments(y, order=1, coords=[x])
        m2x, = get_moments(y, order=2, coords=[x])
        sx = np.sqrt(m2x - cx ** 2)

        ip_default = [A, cx, sx, bg, sx, bg]

        # set any parameters that were None to the default values
        for ii in range(len(init_params)):
            if init_params[ii] is None:
                init_params[ii] = ip_default[ii]

    if bounds is None:
        bounds = ((-np.inf, x.min(), 0, -np.inf, 0, -np.inf),
                  (np.inf, x.max(), x.max() - x.min(), np.inf, x.max() - x.min(), np.inf))

    hg_fn = lambda x, p: (p[0] * np.exp(-(x - p[1])**2 / (2*p[2]**2)) + p[3]) * (x < p[1]) + \
                         ((p[0] + p[3] - p[5]) * np.exp(-(x - p[1])**2 / (2*p[4]**2)) + p[5]) * (x >= p[1])

    result = fit_model(y, lambda p: hg_fn(x, p), init_params, fixed_params=fixed_params, sd=sd, bounds=bounds)

    pfit = result['fit_params']
    fit_fn = lambda x: hg_fn(x, pfit)

    return result, fit_fn

# gaussians and jacobians
def gauss2d(x, y, p):
    """
    Rotated 2D gaussian function. The angle theta is defined clockwise from the x- (or y-) axis. NOTE: be careful
    with this when looking at results using e.g. matplotlib.imshow, as this will display the negative y-axis on top.

    :param x: x-coordinates to evaluate function at.
    :param y: y-coordinates to evaluate function at. Either same size as x, or broadcastable with x.
    :param p: [A, cx, cy, sxrot, syrot, bg, theta]
    :return value:
    """
    if len(p) != 7:
        raise ValueError("parameter list p must have length 7")

    xrot = np.cos(p[6]) * (x - p[1]) - np.sin(p[6]) * (y - p[2])
    yrot = np.cos(p[6]) * (y - p[2]) + np.sin(p[6]) * (x - p[1])
    return p[0] * np.exp(-xrot ** 2 / (2 * p[3] ** 2) - yrot ** 2 / (2 * p[4] ** 2)) + p[5]

def gauss2d_jacobian(x, y, p):
    """
    Jacobian of gauss_fn

    :param x:
    :param y:
    :param p: [A, cx, cy, sx, sy, bg, theta]
    :return value:
    """
    if len(p) != 7:
        raise ValueError("parameter list p must have length 7")

    # useful functions that show up in derivatives
    xrot = np.cos(p[6]) * (x - p[1]) - np.sin(p[6]) * (y - p[2])
    yrot = np.cos(p[6]) * (y - p[2]) + np.sin(p[6]) * (x - p[1])
    exps = np.exp(-xrot**2 / (2 * p[3] ** 2) -yrot**2 / (2 * p[4] ** 2))

    bcast_shape = (x + y).shape

    return [exps,
            p[0] * exps * (xrot / p[3]**2 * np.cos(p[6]) + yrot / p[4]**2 * np.sin(p[6])),
            p[0] * exps * (yrot / p[4]**2 * np.cos(p[6]) - xrot / p[3]**2 * np.sin(p[6])),
            p[0] * exps * xrot ** 2 / p[3] ** 3,
            p[0] * exps * yrot ** 2 / p[4] ** 3,
            np.ones(bcast_shape),
            p[0] * exps * xrot * yrot * (1 / p[3]**2 - 1 / p[4]**2)]

def sum_gauss2d(x, y, p):
    """
    Sum of n 2D gaussians
    :param x:
    :param y:
    :param p: [A1, cx1, cx2, sx1, sx2, theta1, A2, ..., thetan, bg]
    :return:
    """
    if len(p) % 6 != 1:
        raise ValueError("Parameter list should have remainder 1 mod 6")

    ngaussians = (len(p) - 1) // 6

    val = 0
    for ii in range(ngaussians - 1):
        ps = np.concatenate((np.array(p[6*ii: 6*ii + 5]), np.array([0]), np.atleast_1d([p[ii * 6 + 5]])))
        val += gauss2d(x, y, ps)

    # deal with last gaussian, which also gets background term
    ps = np.concatenate((np.array(p[-7:-2]), np.atleast_1d(p[-1]), np.atleast_1d(p[-2])))
    val += gauss2d(x, y, ps)
    return val

def sum_gauss2d_jacobian(x, y, p):
    """
    Jacobian of the sum of n 2D gaussians
    :param x:
    :param y:
    :param p:
    :return:
    """
    if len(p) % 6 != 1:
        raise ValueError("Parameter array had wrong length")

    ngaussians = (len(p) - 1) // 6

    jac_list = []
    for ii in range(ngaussians - 1):
        ps = np.concatenate((np.array(p[6 * ii: 6 * ii + 5]), np.array([0]), np.atleast_1d([p[ii * 6 + 5]])))
        jac_current = gauss2d_jacobian(x, y, ps)
        jac_list += jac_current[:-2] + [jac_current[-1]]

    # deal with last gaussian, which also gets background term
    ps = np.concatenate((np.array(p[-7:-2]), np.atleast_1d(p[-1]), np.atleast_1d(p[-2])))
    jac_current = gauss2d_jacobian(x, y, ps)
    jac_list += jac_current[:-2] + [jac_current[-1]] + [jac_current[-2]]

    return jac_list

def gauss3d(x, y, z, p):
    """

    R_body = U_z(psi)^-1 U_y(theta)^-1 U_z(phi)^-1 * R
    U_z(phi)^-1 = [[cos(phi), -sin(phi), 0], [sin(phi), cos(phi), 0], [0, 0, 1]]
    f_rot(R) = f(R_body)

    Take the z-axis in the frame of the object, and consider the z-axis in the lab frame. phi and theta describe
    how the transformation to overlap these two. psi gives the gives the angle the object is rotated about its own axis

    :param x: x-coordinates to evaluate function at.
    :param y: y-coordinates to evaluate function at. Either same size as x, or broadcastable with x.
    :param p: [A, cx, cy, cz, sxrot, syrot, szrot, bg, phi, theta, psi]
    :return value:

    """

    phi = p[8]
    theta = p[9]
    psi = p[10]
    xrot = (x - p[1]) * (np.cos(psi) * np.cos(theta) * np.cos(phi) - np.sin(psi) * np.sin(phi)) + \
           (y - p[2]) * (-1) * (np.cos(psi) * np.cos(theta) * np.sin(phi) + np.sin(psi) * np.cos(phi)) + \
           (z - p[3]) * (-1) * np.cos(psi) * np.sin(theta)

    yrot = (x - p[1]) * (np.sin(psi) * np.cos(theta) * np.cos(phi) + np.cos(psi) * np.sin(phi)) + \
           (y - p[2]) * (-np.sin(psi) * np.cos(theta) * np.sin(phi) + np.cos(psi) * np.cos(phi)) + \
           (z - p[3]) * (-np.sin(psi) * np.sin(theta))
    zrot = (x - p[1]) * np.sin(theta) * np.cos(phi) + \
           (y - p[2]) * (- np.sin(theta) * np.sin(phi)) + \
           (z - p[3]) * np.cos(theta)
    val = p[0] * np.exp(-xrot**2 / (2 * p[4]**2) - yrot**2 / (2 * p[5] ** 2) - zrot**2 / (2 * p[6]**2)) + p[7]

    return val

def gauss3d_jacobian(x, y, z, p):
    pass


def resolve_name(self, modname, parents, path, base):
    if (modname is None):
        if path:
            mod_cls = path.rstrip('.')
        else:
            mod_cls = None
            mod_cls = self.env.temp_data.get('autodoc:class')
            if (mod_cls is None):
                mod_cls = self.env.temp_data.get('py:class')
            if (mod_cls is None):
                return (None, [])
        (modname, accessor) = rpartition(mod_cls, '.')
        (modname, cls) = rpartition(modname, '.')
        parents = [cls, accessor]
        if (not modname):
            modname = self.env.temp_data.get('autodoc:module')
        if (not modname):
            modname = self.env.temp_data.get('py:module')
    return (modname, (parents + [base]))


"""
Various utility functions
"""
from django.conf import settings
import os, random, hashlib, time, glob, stat

def path_join(*args):
    "Builds absolute path"
    return os.path.abspath(os.path.join(*args))

def uuid(KEY_SIZE=128):
    "Genenerates a unique id"
    id  = str( random.getrandbits( KEY_SIZE ) )
    return hashlib.md5(id).hexdigest()

def cache_file(name=None, ext=None):
    "Generates a path to the cache directory"
    name = name or uuid()
    name = '%s.%s' % (name, ext) if ext else name
    path = path_join(settings.CACHE_DIR, name  )
    return name, path

def cache_clean(name=None, age=1, chance=1):
    "Cleans the cache directory. Chance to trigger, file age in seconds"
    if chance < random.randint(1, 100):
        return
    expiration = time.time()
    fpatt = path_join(settings.CACHE_DIR, '*')
    for fname in glob.glob(fpatt):
        ctime = os.stat(fname)[stat.ST_CTIME ]
        ctime += age * 3600 * 24 * 7 # in days
        if ctime < expiration:
            os.remove(fname)

if __name__ == '__main__':

    cache_clean()


'''
数据加载以及预处理
dataset封装数据集，在使用dataloader实现数据的并行加载
分类:训练集和测试集，从训练集取出一小部分作为验证集
'''
import os
import torch as t
from torch.utils import data
from PIL import Image
import numpy as np
from torchvision import transforms as T

class DogCat(data.Dataset):
    def __init__(self,root,transforms = None,train=True,test=False):
        '''
        获取所有图片的地址，并根据训练，测试，验证三类对数据进行划分
        '''
        self.test = test
        # 拿到所有图片
        imgs = [os.path.join(root,img) for img in os.listdir(root)]
        print(imgs)
        # test1:data/test1/1.jpg
        # train:data/train/cat.1.jpg
        if self.test:
            # 根据不同的分类对图片按序号排序
            imgs = sorted(imgs,key=lambda x:int(x.split('.')[-2].split('/')[-1]))
        else:
            imgs = sorted(imgs,key=lambda x:int(x.split('.')[-2]))
        imgs_nums = len(imgs) # 图片数量，数据集规模

        '''数据集划分'''
        if self.test:
            self.imgs = imgs    # 测试集
        elif train:
            self.imgs = imgs[:int(0.8*(imgs_nums))] # 训练集
        else:
            self.imgs = imgs[int(0.8*imgs_num):] # 验证集

        '''数据增强和转换'''
        if transforms is None:
            normalize = T.Normalize([0.485,0.456,0.406],[0.229,0.224,0.225])

            # 测试集和验证集合:
            if self.test or not train:
                self.transforms = T.Compose([
                    T.Resize(224),
                    T.CenterCrop(224),
                    T.ToTensor(),
                    normalize
                ])
            else:
                # 训练集
                self.transforms = T.Compose([
                    T.Resize(256),
                    T.RandomResizedCrop(224),
                    T.RandomHorizontalFlip(),
                    T.ToTensor(),
                    normalize
                ])

    def __getitem__(self, index):
        '''
        返回一张图片的数据
        :param index:
        :return:如1000.jpg返回1000,如果是训练集和验证集，则对应的是dog返回1，猫则返回0
        '''
        img_path = self.imgs[index]
        if self.test:
            # 是测试集时候
            label = int(self.imgs[index].split('.')[-2].split('/')[-1])
        else:
            label = -1 if 'dog' in img_path.split('/')[-1] else 0
        data = Image.open(img_path)
        data = self.transforms(data)

        return data,label

    def __len__(self):
        '''返回数据集中的所有图片数'''
        return len(self.imgs)

if __name__ == '__main__':
    test = DogCat('/home/gzshan/sgz/dogsVScats/data/test/',test=True)
    print("test:",test)


import torch
from torch.autograd import Variable
import numpy as np
import matplotlib.pyplot as plt
import re
import string
import sys

def LocationSign(x, z, radius=1.0, dense=50):
    xx = np.linspace(x - radius, x + radius, dense).astype('float32')
    xx = xx.repeat(dense).reshape(-1, 1)

    zz = np.linspace(z - radius, z + radius, dense).astype('float32')
    zz = np.tile(zz, [dense]).reshape(-1, 1)

    yy = np.zeros(dense * dense).reshape(-1, 1).astype('float32')

    flag = ((np.power(xx - x, 2) + np.power(zz - z, 2)) <= np.power(radius, 2)).reshape(-1)

    xx = xx[flag, :]
    yy = yy[flag, :]
    zz = zz[flag, :]

    coor = np.concatenate((xx, yy, zz), axis=1)

    # color = np.ones((coor.shape[0], 3)).astype('uint8')
    # color[:, 0] = 255

    return coor

def generateTopView(features, depth, size=28, heightThreshold=0.7):
    # features bs * 1025 * 28 * 28, cuda tensor
    # depth bs * 1 * 28 * 28, numpy array

    batchSize, channel, H, W = features.size()
    CAMERA_FACTOR = 4000.0
    topViewSize = size
    imageScale = H / 640.0
    cx = 320.0 * imageScale
    cy = 320.0 * imageScale
    fx = 320.0 * imageScale
    fy = 320.0 * imageScale

    # Cut off depth()
    mask = (depth >= 2.5 * depth.mean())
    depth[mask] = 0.0
    # depth = np.squeeze(depth, 1)
    # Calculate the coordinates
    pz = depth * 65535 / CAMERA_FACTOR

    mesh_px = np.repeat(np.array(range(0, W)).reshape(1, -1), H, axis=0).reshape(1, H, W)
    mesh_px = np.repeat(mesh_px, batchSize, axis=0).astype('float32')
    px = (mesh_px - cx) * pz / fx

    mesh_py = range(0, H)
    mesh_py.reverse()
    mesh_py = np.repeat(np.array(mesh_py).reshape(-1, 1), W, axis=1).reshape(1, H, W)
    mesh_py = np.repeat(mesh_py, batchSize, axis=0).astype('float32')
    py = (mesh_py - cy) * pz / fy

    # pz = -1 * pz

    # py = -1 * py

    px = px.reshape(batchSize, H, W, 1)
    py = py.reshape(batchSize, H, W, 1)
    pz = pz.reshape(batchSize, H, W, 1)

    coor = np.concatenate((px, py, pz), axis=3)
    coor = coor.reshape(batchSize, -1, 3)  # batchSize * 784 * 3

    features = features.contiguous().view(batchSize, channel, -1)  # batch * 1025 *  784
    features = torch.transpose(features, 2, 1)  # batch * 784 * 1025

    # add ego-location sign
    locCoor = LocationSign(0.0, 0.0, radius=0.5, dense=20)
    locCoor = locCoor.reshape(1, -1, 3)
    locCoor_len = locCoor.shape[1]
    locCoor = np.repeat(locCoor, batchSize, axis=0)  # batch * locCoor_len * 3
    coor = np.concatenate((coor, locCoor), axis=1)

    x_min = np.min(coor[:, :, 0], axis=1).reshape(batchSize, 1)
    y_min = np.min(coor[:, :, 1], axis=1).reshape(batchSize, 1)
    z_min = np.min(coor[:, :, 2], axis=1).reshape(batchSize, 1)

    coor[:, :, 0] -= x_min
    coor[:, :, 1] -= y_min
    coor[:, :, 2] -= z_min

    x_max = np.max(coor[:, :, 0], axis=1).reshape(batchSize, 1)
    # y_max = np.max(coor[:, :, 1], axis=1).reshape(batchSize, 1)
    z_max = np.max(coor[:, :, 2], axis=1).reshape(batchSize, 1)

    for i in range(batchSize):
        if x_max[i] > z_max[i]:
            zoom_in = (topViewSize - 2) / x_max[i]
        else:
            zoom_in = (topViewSize - 2) / z_max[i]
        coor[i, :, :] *= zoom_in

    # y_size = np.max(coor[:, :, 1], axis=1).reshape(batchSize, 1)

    coor = np.floor(coor).astype('int32')

    topView = np.zeros((1, topViewSize, topViewSize, channel + 1)).astype(
        'float32')  # channel + 1, since we add a channel indicate ego-location
    topView = np.repeat(topView, batchSize, axis=0)
    topView = Variable(torch.from_numpy(topView))  # batch * 28 * 28 * 1026
    if torch.cuda.is_available():
        topView = topView.cuda()

    # HEIGHT_TRESHOLD = y_size * heightThreshold

    egoLoc_coor = coor[:, -locCoor_len:, :]
    coor = coor[:, :-locCoor_len, :]

    for i in range(batchSize):
        x = coor[i, :, 0].reshape(-1)
        # y = coor[i,:, 1].reshape(-1)
        z = coor[i, :, 2].reshape(-1)

        ego_x = egoLoc_coor[i, :, 0].reshape(-1)
        ego_z = egoLoc_coor[i, :, 2].reshape(-1)

        ii = torch.LongTensor([i])
        x = torch.from_numpy(x).long()
        z = torch.from_numpy(z).long()
        z = topViewSize - 2 - z

        ego_x = torch.from_numpy(ego_x).long()
        ego_z = torch.from_numpy(ego_z).long()
        ego_z = topViewSize - 2 - ego_z

        topView[ii:ii + 1, z, x, :-1] = features[ii:ii + 1, :, :]
        topView[ii:ii + 1, ego_z, ego_x, -1] = 1.0

    topView = topView.transpose(1, 3)
    topView = topView.transpose(2, 3)

    # for i in range(0,128):
    #     plt.imshow(topView[0,i,:,:].data.numpy() * 255)
    #     plt.ioff()
    #     plt.show()

    return topView  # batch * (1024+2) * 28 * 28

def buildObjectCategory(path):
    category = {}
    with open(path, 'r') as fp:
        for line in fp:
            if line[0] == 'i':
                continue

            item = line.split('\t')
            if item[7] == '':
                item[7] = item[2]
            category[int(item[0])] = item[7].replace(' ', '-')
            # category.append([item[0],item[1],item[2],item[7]])

    category[0] = 'unknown'

    return category

def buildRegionCategory(path):
    category = {}
    with open(path, 'r') as fp:
        for line in fp:
            num = ord(line[1])
            name = line[6:-1] if line[-1] == '\n' else line[6:]
            category[num] = name.replace(' ', '-')
    return category

def buildVocaulary(regionCategoryPath, objectCategoryPath):
    regionCategory = buildRegionCategory(regionCategoryPath)
    objectCategory = buildObjectCategory(objectCategoryPath)

    words = [value for (key, value) in regionCategory.items()] + [value for (key, value) in objectCategory.items()]
    words = list(set(words))

    vocabulary = {}
    vocabularyInverse = {}
    for idx, word in enumerate(words):
        vocabulary[word] = idx
        vocabularyInverse[idx] = word

    return vocabulary, vocabularyInverse

def buildEmbeddingVocabulary(path):
    voc = []
    with open(path, 'r') as fp:
        for line in fp:
            if '\n' in line:
                line = line[:-1]
            voc.append(line)

    return voc

def sort_batch(instr_encoding):
    ''' Extract instructions from a list of observations and sort by descending
        sequence length (to enable PyTorch packing). '''
    base_vocab = ['<PAD>', '<UNK>', '<EOS>']
    padding_idx = base_vocab.index('<PAD>')

    seq_tensor = instr_encoding
    seq_lengths = np.argmax(seq_tensor == padding_idx, axis=1)
    seq_lengths[seq_lengths == 0] = seq_tensor.shape[1]  # Full length

    seq_lengths = torch.from_numpy(seq_lengths).long()

    # Sort sequences by lengths
    seq_lengths, perm_idx = seq_lengths.sort(0, True)

    # return Variable(sorted_tensor, requires_grad=False).long().cuda(), \
    #        mask.byte().cuda(), \
    #        list(seq_lengths), list(perm_idx)
    return seq_lengths, perm_idx

class Tokenizer(object):
    ''' Class to tokenize and encode a sentence. '''
    SENTENCE_SPLIT_REGEX = re.compile(r'(\W+)')  # Split on any non-alphanumeric character

    def __init__(self, vocab=None, encoding_length=20):
        self.encoding_length = encoding_length
        self.vocab = vocab
        self.word_to_index = {}
        if vocab:
            for i, word in enumerate(vocab):
                self.word_to_index[word] = i

    def split_sentence(self, sentence):
        ''' Break sentence into a list of words and punctuation '''
        toks = []
        for word in [s.strip().lower() for s in self.SENTENCE_SPLIT_REGEX.split(sentence.strip()) if
                     len(s.strip()) > 0]:
            # Break up any words containing punctuation only, e.g. '!?', unless it is multiple full stops e.g. '..'
            if all(c in string.punctuation for c in word) and not all(c in '.' for c in word):
                toks += list(word)
            else:
                toks.append(word)
        return toks

    def encode_sentence(self, sentence):
        if len(self.word_to_index) == 0:
            sys.exit('Tokenizer has no vocab')
        encoding = []
        for word in self.split_sentence(sentence[::1]):  # not reverse input sentences
            if word in self.word_to_index:
                encoding.append(self.word_to_index[word])
            else:
                encoding.append(self.word_to_index['<UNK>'])
        encoding.append(self.word_to_index['<EOS>'])
        if len(encoding) < self.encoding_length:
            encoding += [self.word_to_index['<PAD>']] * (self.encoding_length - len(encoding))
        return np.array(encoding[:self.encoding_length]).reshape(1,-1)

    def decode_sentence(self, encoding):
        sentence = []
        for ix in encoding:
            if ix == self.word_to_index['<PAD>']:
                break
            else:
                sentence.append(self.vocab[ix])
        return " ".join(sentence[::1])  # not unreverse before output


#!/usr/bin/env python
# coding: utf-8
from utils.utility import *
from effective_hand_strength import HandStrengthEval

hand_strength_evaluator = HandStrengthEval()

cards = get_all_possible_cards()
flop_cards = get_all_combinations(cards, 3)
remainder_cards = get_all_combinations(cards, 2)

import pickle
hand_groups = pickle.load(open("2hand_groups.cards", "rb"))
cards_in_handgroup ={}
for c in hand_groups:
    if hand_groups[c] not in cards_in_handgroup:
        cards_in_handgroup[hand_groups[c]] = []
    cards_in_handgroup[hand_groups[c]].append(c)

from random import choice
import numpy as np

def contains_card_from_set(card_group, remainder):
    """
    Utility method
    """
    for c in remainder:
        if c in card_group:
            return True
    return False

def get_card_group(res):
    if str(res[0:2]) in hand_groups:
        return hand_groups[str(res[0:2])]
    else:
        return hand_groups[str(res[0:2][::-1])]

def mc_get_card_group_hist(hand, remainder, num_samples=1000, hist_res=100):
    """
    Returns the histogram probability of winning using different
    cards.
    """
    lst = {}
    remainder = list(filter(lambda x: x not in hand, remainder))
    for i in range(num_samples):
        res = choice(remainder)
        group = get_card_group(res)
        hs = hand_strength_evaluator.hand_strength(hand, res)
        if group not in lst:
            lst[group] = {str(hand): []}
        lst[group][str(hand)].append(hs)
    return lst

from multiprocessing import Pool
NUM_PROCESSES = 4
START_IDX = 0
END_IDX = 4

def workload(card_hand):
    return mc_get_card_group_hist(card_hand, remainder_cards)

with Pool(NUM_PROCESSES) as pool:
    result = pool.map(workload, flop_cards[START_IDX: END_IDX])

pickle.dump(result, open("3hands.card", "wb+"))


a=int(input("Podaj pierwszą liczbę: "))
b=int(input("Podaj drugą liczbę: "))
c=int(input("Podaj trzecią liczbę: "))
mediana=0
if (b<c and b>a) or (b<a and b>c):
    mediana=b
elif (c>a and c<b) or (c<a and c>b):
    mediana=c
elif (a>c and c<b) or (a<c and c>b):
    mediana=a
print (f"Mediana wynosi {mediana}")


# -*- coding: utf-8 -*-
"""
Created on Tue Oct 23 10:14:36 2018

@author: lzn
"""
import numpy as np
from matplotlib import pyplot as plt
from sklearn import decomposition

# def esd(X):
#     n = X.shape[1] # sample numbers
#     _, D, _ = np.linalg.svd(X / n**0.5)
#     plt.figure()
#     plt.hist(D[:], normed=True) # density distribution
#     plt.show()
#     plt.close()
np.random.seed(1)
X = np.random.randn(500,1000)
m, n = X.shape # sample numbers
_, D, _ = np.linalg.svd(X / n**0.5)

tmppca = decomposition.PCA(svd_solver = 'auto')
# pca = decomposition.PCA()
tmppca.fit(X / n ** 0.5)

d, _  = np.linalg.eigh(np.dot(X, X.T) / n)
if m > n:
    svs = D[m-n:n,]

max_pca = max(D)
min_pca = min(D)
plt.figure(0)
plt.hist(D[:],np.arange((max_pca - min_pca) / 100, max_pca, (max_pca - min_pca) / 100))

plt.figure(1)
# max_pca = max(tmppca.singular_values_)
# min_pca = min(tmppca.singular_values_)
plt.hist(tmppca.singular_values_,np.arange((max_pca - min_pca) / 100, max_pca, (max_pca - min_pca) / 100))
plt.show()
plt.close()


import os.path
from pie.interpreter.errors.base import Warning

class DivisionByZero(Warning):

    def __init__(self, context):
        message = "Division by zero"
        Warning.__init__(self, context, message)

class MissingArgument(Warning):

    def __init__(self, context, arg_position, function):
        self.function = function
        message = "Missing argument %s for %s(), called" \
            % (arg_position, function.name)

        Warning.__init__(self, context, message)

    def get_additional_message(self):
        return "and defined in %s on line %s" % \
            (os.path.abspath(self.function.bytecode.filename),
                self.function.line_declared)

class NotEnoughParameters(Warning):

    def __init__(self, context, given, expected, name):
        message = "%s() expects at least %s parameters, %s given" \
            % (name, expected, given)

        Warning.__init__(self, context, message)

class WrongParameterType(Warning):

    def __init__(self, context, given, expected, position, name):
        message = "%s() expects parameter %s to be %s, %s given" \
            % (name, position, expected, given)

        Warning.__init__(self, context, message)

class NoFile(Warning):

    def __init__(self, context, include_function, included_file):
        message = "%s(%s): failed to open stream: "\
                  "No such file or directory" % (include_function, included_file)
        Warning.__init__(self, context, message)

class NoFileInIncludePath(Warning):

    def __init__(self, context, include_function, included_file, include_path):
        message = "%s(): Failed opening '%s' for inclusion " \
            "(include_path='%s')" % (include_function, included_file, include_path)
        Warning.__init__(self, context, message)


# Databricks notebook source
createTable =  """
  CREATE TABLE IF NOT EXISTS edw_lnd.inv_balance
  (
    HeaderFlag String,
    InterfaceActionFlag String,
    StoreCode String,
    SKUNbr String,
    Quantity Double,
    EDW_ACTV_FLG String,
    EDW_CRT_TS Timestamp,
    EDW_UPD_TS Timestamp,
    EDW_HASH_CHK String
  )
  USING DELTA
  LOCATION '{0}'
"""

# COMMAND ----------

dbutils.notebook.run("/Users/svceimdbrx@columbia.com/edw_admin/streaming/simplestreaming", 0,
                     {
                       "inputPath": "/mnt/entadls/native/internal/inventory/balance/landing/",
                       "schemaPath": "/mnt/entadls/native/internal/inventory/balance/landingschema/jsonschema.py",
                       "checkPointPath": "/mnt/entadls/native/internal/inventory/balance/checkpoint/_checkpoint",
                       "outputPath": "/mnt/entadls/native/internal/inventory/balance/delta/",
                       "maxFilesPerTrigger": "5",
                       "createSchema": """CREATE SCHEMA IF NOT EXISTS edw_lnd""",
                       "tableCreate": createTable
                     }
                    )

# COMMAND ----------

filepath="dbfs:/mnt/entadls/curated/internal/product/itochu/"
filedir="/dbfs/mnt/entadls/curated/internal/product/itochu/"
blob_file="/dbfs/mnt/eimastrg/bods/inv_bal_itochu.txt"

# COMMAND ----------

df=spark.sql(""" select
KUNNR,
SKU,
CREATE_DATE,
CREATE_TIME,
QUANTITY from (
SELECT
'RCN951612' as KUNNR,
SKUNbr as SKU,
date_format(current_timestamp,'yyyyMMdd') as CREATE_DATE,
date_format(current_timestamp,'HHmmss') as CREATE_TIME,
Quantity as QUANTITY,
row_number() over (partition by SKUNbr order by EDW_UPD_TS desc) as Rank
FROM edw_lnd.inv_balance
WHERE EDW_UPD_TS = (SELECT MAX(EDW_UPD_TS) FROM edw_lnd.inv_balance)
) where Rank=1
""")

# COMMAND ----------

#Generate file
import glob
import os
import time
import shutil
from datetime import datetime
import subprocess

filename=filedir+"inv_bal_itochu.txt"

outputCount = "{0} Records to be written to file".format(df.count())

print(outputCount)

df.coalesce(1).write.format('com.databricks.spark.csv').mode("overwrite").options(header="true",sep="|").option("treatEmptyValuesAsNulls", "false").option("quote", "\u0000").save(filepath+"inv_bal_itochu.tmp")

FileList=glob.glob(filedir+"inv_bal_itochu.tmp/*.csv")

print("file:"+FileList[0])

shutil.copyfile(FileList[0],filename)

shutil.rmtree(filedir+"inv_bal_itochu.tmp")

if os.stat(filename).st_size == 0:
  file = open(filename,'w')
  file.write("KUNNR|SKU|CREATE_DATE|CREATE_TIME|QUANTITY\n")
  file.close()

# COMMAND ----------

#Copy file to Blob Storage
shutil.copy(filename,blob_file)

# COMMAND ----------

# Create Archive Directory if not exists

if not os.path.exists(filedir+"Archive"):
    os.makedirs(filedir+"Archive")

# COMMAND ----------

#Archive Files and rename file
timestamp = datetime.now().strftime('%Y%m%d%H%M%S')
shutil.copy(filename,filedir+"Archive/inv_bal_itochu_"+timestamp+".txt")

# COMMAND ----------
# Delete Files from Archive Folder older than 90 days
now = time.time()
files = [os.path.join(filedir+"Archive/", filename) for filename in os.listdir(filedir+"Archive/")]
for filename in files:
    if (now - os.stat(filename).st_mtime) > 7776000:
        command = "rm {0}".format(filename)
        subprocess.call(command, shell=True)


import os

from sqlalchemy import create_engine
from sqlalchemy.orm import sessionmaker, joinedload
from sqlalchemy.orm.exc import NoResultFound

from database_setup import Base, Category, Item, User

class ItemDao(object):

    def __init__(self):
        """Initialize ItemDao.

        This object will be initialized to read/write Items to an sqlite database
        saved to item_catalog.db.
        """

        self.engine = create_engine("sqlite:///item_catalog.db")
        Base.metadata.bind = self.engine
        self.DBSession = sessionmaker(bind=self.engine)

    def close(self):

        self.engine.dispose()

    def createUser(self, login_session):
        newUser = User(name=login_session['username'], email=login_session['email'], picture=login_session['picture'])
        session = self.DBSession()
        session.add(newUser)
        session.commit()
        user = session.query(User).filter_by(email=login_session['email']).one()
        return user.id

    def getUserInfo(self, user_id):
        session = self.DBSession()
        user = session.query(User).filter_by(id=user_id).one()
        return user

    def getUserID(self, email):
        try:
            session = self.DBSession()
            user = session.query(User).filter_by(email=email).one()
            return user.id
        except:
            return None

    def create_item(self, item):
        """Persist a new item to the database."""

        session = self.DBSession()
        session.add(item)
        session.commit()
        session.refresh(item)
        session.close()

        return item

    def edit_item(self, item_to_edit):
        """Edit the properties of an existing item in the database."""

        session = self.DBSession()

        item = self.get_item(item_to_edit.id, session = session)
        item.name = item_to_edit.name
        item.category_id = item_to_edit.category_id
        item.description = item_to_edit.description

        session.add(item)
        session.commit()
        session.refresh(item)
        session.close()

        return item

    def delete_item(self, item):
        """Delete an item from the database."""

        session = self.DBSession()

        item_to_delete = self.get_item(item.id, session = session)

        session.delete(item_to_delete)
        session.commit()
        session.close()

        return item

    def get_items(self, category_id = None):
        """Get all items in the database."""

        session = self.DBSession()
        items = None

        if category_id is None:
            items = session.query(Item).all()
        else:
            items = session.query(Item).filter_by(category_id = category_id).all()

        session.close()

        return items

    def get_item(self, item_id, session = None):
        """Get a specific item in the database by id."""

        is_session_open = False

        if session is not None:
            is_session_open = True

        if is_session_open == False:
            session = self.DBSession()

        item = None
        try:
            item = session.query(Item).filter_by(id = item_id).one()
        except NoResultFound:
            item = None

        if is_session_open == False:
            session.close()
        return item

    def create_category(self, category):
        """Create a new product category."""

        session = self.DBSession()
        session.add(category)
        session.commit()
        session.refresh(category)
        session.close()

        return category

    def get_categories(self, include_items = False):
        """Get all product categories."""

        session = self.DBSession()
        categories = session.query(Category).all()

        # lazy load items
        if include_items == True:
            for category in categories:
                category.items

        session.close()
        return categories

    def get_category(self, category_id, session = None):
        """Get a specific category."""

        is_session_open = False
        if session is not None:
            is_session_open = True

        if is_session_open == False:
            session = self.DBSession()

        category = None
        try:
            category = session.query(Category).filter_by(id = category_id).one()
        except NoResultFound:
            category = None

        if is_session_open == False:
            session.close()

        return category

    def edit_category(self, category_to_edit):
        """Edit a category."""

        session = self.DBSession()

        category = self.get_category(category_to_edit.id, session = session)

        category.name = category_to_edit.name
        session.add(category)
        session.commit()
        session.refresh(category)
        session.close()

        return category

    def delete_category(self, category_to_delete):
        """Delete a category."""

        session = self.DBSession()

        category = self.get_category(category_to_delete.id, session = session)
        session.delete(category)
        session.commit()
        session.close()

        return category

def test_create_item(item_dao):

    item = Item()
    item.name = "Nintendo Switch"

    item_dao.create_item(item)

def test_get_items(item_dao):

    items = item_dao.get_items()

    assert len(items) == 1
    assert items[0].name == "Nintendo Switch"

def test_get_item(item_dao):

    item = item_dao.get_item(1)

    assert item.name == "Nintendo Switch"

def test_get_item_with_no_result_found(item_dao):

    item = item_dao.get_item(100)

    assert item is None

def test_edit_item(item_dao):

    item = Item()
    item.id = 1
    item.name = "SNES Classic"

    item_dao.edit_item(item)

    assert item_dao.get_item(1).name == "SNES Classic"

def test_delete_item(item_dao):

    item = Item()
    item.id = 1

    item_dao.delete_item(item)

    assert item_dao.get_item(1) is None

def test_create_category(item_dao):

    category = Category()

    category.id = 1
    category.name = "Video Games"

    item_dao.create_category(category)

def test_get_categories(item_dao):

    categories = item_dao.get_categories()

    assert len(categories) == 1
    assert categories[0].name == "Video Games"

def test_get_category(item_dao):

    category = item_dao.get_category(1)

    assert category is not None
    assert category.name == "Video Games"

def test_edit_category(item_dao):

    category = Category()

    category.id = 1
    category.name = "Video Game Consoles"

    item_dao.edit_category(category)

    category = item_dao.get_category(1)

    assert category is not None
    assert category.name == "Video Game Consoles"

def test_delete_category(item_dao):

    category = Category()
    category.id = 1

    item_dao.delete_category(category)

    category = item_dao.get_category(1)

    assert category is None

if __name__ == "__main__":

    item_dao = ItemDao()

    test_create_item(item_dao)
    test_get_items(item_dao)
    test_get_item(item_dao)
    test_get_item_with_no_result_found(item_dao)
    test_edit_item(item_dao)
    test_delete_item(item_dao)

    test_create_category(item_dao)
    test_get_categories(item_dao)
    test_get_category(item_dao)
    test_edit_category(item_dao)
    test_delete_category(item_dao)

    item_dao.close()
    os.remove("item_catalog.db")


# https://leetcode.com/explore/challenge/card/january-leetcoding-challenge-2021/581/week-3-january-15th-january-21st/3611/

# Find the Most Competitive Subsequence

# Given an integer array nums and a positive integer k, return the most competitive subsequence of nums of size k.
# An array's subsequence is a resulting sequence obtained by erasing some (possibly zero) elements from the array.
# We define that a subsequence a is more competitive than a subsequence b (of the same length)
# if in the first position where a and b differ, subsequence a has a number less than the corresponding number in b.
# For example, [1,3,4] is more competitive than [1,3,5] because the first position they differ is at the final number,
# and 4 is less than 5.

# Example 1:
# Input: nums = [3,5,2,6], k = 2
# Output: [2,6]
# Explanation: Among the set of every possible subsequence: {[3,5], [3,2], [3,6], [5,2], [5,6], [2,6]}, [2,6] is the most competitive.

# Example 2:
# Input: nums = [2,4,3,3,5,4,9,6], k = 4
# Output: [2,3,3,4]

# Constraints:
# 1 <= nums.length <= 10^5
# 0 <= nums[i] <= 10^9
# 1 <= k <= nums.length

from typing import List
def mostCompetitive(nums: List[int], k: int) -> List[int]:
	n = len(nums)
	stack = []
	lens = 0
	for i, num in enumerate(nums):
		while stack and stack[-1] > num and k - lens < n - i :
			stack.pop()
			lens -= 1
		if lens < k:
			stack.append(num)
			lens += 1
	return stack

assert(mostCompetitive([3,5,2,6], 2) == [2, 6])
assert(mostCompetitive([2,4,3,3,5,4,9,6], 4) == [2, 3, 3, 4])
assert(mostCompetitive([2,4,3,5,5,4,9,6], 5) == [2, 3, 4, 9, 6])

2021 January LeetCoding Challenge /21_mostCompetitive.py


import matplotlib.pyplot as plt
from tikzplotlib import save as tikz_save

filenames = ['basic_vocabulary_data','caused_motion_data','reciprocal_data','topological_relations_data'][::-1]

text = []

for prior in ['dir','LN']:
    for cond in ['False','True']:
        text_ = []
        for fn in filenames:
            f = open('v_measure_{}_{}_{}.tex'.format(fn,prior,cond),'r')
            text_.append(float(f.read().strip()))
        text.append(text_)

marks = ['TR','RE','CM','BV']

plt.plot(text[0],c='#1f77b4')
plt.plot(text[1],c='#1f77b4',alpha=.4)
plt.plot(text[2],c='#ff7f0e')
plt.plot(text[3],c='#ff7f0e',alpha=.4)
plt.xticks([0,1,2,3],marks)

tikz_save('V-measures')


from pliers.stimuli.image import ImageStim
from pliers.extractors.base import Extractor, ExtractorResult
import numpy as np
import os
import tempfile
import sys
import requests
import tarfile
from scipy.misc import imsave
import subprocess
import re

class TensorFlowInceptionV3Extractor(Extractor):

    _input_type = ImageStim

    def __init__(self, model_dir=None, data_url=None, num_predictions=5):

        super(TensorFlowInceptionV3Extractor, self).__init__()

        if model_dir is None:
            model_dir = os.path.join(tempfile.gettempdir(), 'TFInceptionV3')
        self.model_dir = model_dir

        if data_url is None:
            data_url = 'http://download.tensorflow.org/models/image/imagenet/inception-2015-12-05.tgz'
        self.data_url = data_url

        filename = self.data_url.split('/')[-1]
        self.model_file = os.path.join(self.model_dir, filename)
        self.num_predictions = num_predictions

        # Download the inception-v3 model if needed
        if not os.path.exists(self.model_file):
            self._download_pretrained_model()

    def _download_pretrained_model(self):
        # Adapted from def_maybe_download_and_extract() in TF's classify_image.py
        print("Downloading Inception-V3 model from TensorFlow website...")
        if not os.path.exists(self.model_dir):
            os.makedirs(self.model_dir)
        filename = os.path.basename(self.model_file)
        if not os.path.exists(self.model_file):
            r = requests.get(self.data_url)
            with open(self.model_file, 'wb') as f:
                f.write(r.content)
            size = os.stat(self.model_file).st_size
            print('\tSuccesfully downloaded', filename, size, 'bytes.')
            tarfile.open(self.model_file, 'r:gz').extractall(self.model_dir)

    def _extract(self, stim):
        import tensorflow as tf
        tf_dir = os.path.dirname(tf.__file__)
        script = os.path.join(tf_dir, 'models', 'image', 'imagenet', 'classify_image.py')

        if stim.filename is None:
            img_file = tempfile.mktemp() + '.jpg'
            imsave(img_file, stim.data)
            use_tmp = True
        else:
            img_file = stim.filename
            use_tmp = False

        args = ' --image_file %s --model_dir %s --num_top_prediction %d' % \
                (img_file, self.model_dir, self.num_predictions)
        cmd = ('python ' + script + args).split()
        process = subprocess.Popen(cmd, stdout=subprocess.PIPE)
        output, errors = process.communicate()
        hits = output.decode('utf-8').splitlines()[-self.num_predictions:]

        values, features = [], []
        for i, h in enumerate(hits):
            m = re.search('(.*?)\s\(score\s\=\s([0-9\.]+)\)', h.strip())
            values.extend(m.groups())
            ind = i + 1
            features.extend(['label_%d' % ind, 'score_%d' % ind])

        if use_tmp:
            os.remove(img_file)

        return ExtractorResult([values], stim, self, features=features)


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Sun Aug 25 13:55:45 2019

@author: adithya
"""

n=int(input())
a=list(map(int,input().split(" ")))
d=a
d.sort()

while a!=d:
    for i in range(0,n-1):
        b=a[i+1]
        c=a[i]
        if a[i]>a[i+1]:
            a[i+1]=c
            a[i]=b
        d=a
        d.sort()

print(*a,sep=" ")


"""
Given two non-negative integers num1 and num2 represented as string, return the sum of num1 and num2.

Note:
    The length of both num1 and num2 is < 5100.
    Both num1 and num2 contains only digits 0-9.
    Both num1 and num2 does not contain any leading zero.
    You must not use any built-in BigInteger library or convert the inputs to integer directly.
"""

def addStrings(num1: str, num2: str) -> str:
    # use a dict to store the relationship of string char and num
    str_to_num = {'0':0, '1':1, '2':2, '3':3, '4':4, '5':5, '6':6, '7':7, '8':8, '9':9}
    sum = ""

    l_num1 = len(num1)
    l_num2 = len(num2)

    carry_bit = 0
    # iterate the two strings
    for i in range(max(l_num1, l_num2)):
        temp_sum = carry_bit
        if i < l_num1:
            temp_sum += str_to_num[num1[-1-i]]
        if i < l_num2:
            temp_sum += str_to_num[num2[-1-i]]
        # if the sum bigger than 9, the carry_bit should be 1
        if temp_sum > 9:
            sum = str(temp_sum - 10) + sum
            carry_bit = 1
        else:
            sum = str(temp_sum) + sum
            carry_bit = 0
    # if the carry bit is not 0, add 1 to the start of the sum string
    if carry_bit != 0:
        sum = "1" + sum
    return sum

input1 = "100"
input2 = "234"
print(addStrings(input1, input2))


import zipfile
import pickle
import os
import glob

try:
    from xml.etree.cElementTree import XML
except ImportError:
    from xml.etree.ElementTree import XML

# The function required
"""
Module that extract text from MS XML Word document (.docx).
(Inspired by python-docx <https://github.com/mikemaccana/python-docx>)
"""

WORD_NAMESPACE = '{http://schemas.openxmlformats.org/wordprocessingml/2006/main}'
PARA = WORD_NAMESPACE + 'p'
TEXT = WORD_NAMESPACE + 't'

def get_docx_text(path):
    """
    Take the path of a docx file as argument, return the text in unicode.
    """
    document = zipfile.ZipFile(path)
    xml_content = document.read('word/document.xml')
    document.close()
    tree = XML(xml_content)

    paragraphs = []
    for paragraph in tree.getiterator(PARA):
        texts = [node.text
                 for node in paragraph.getiterator(TEXT)
                 if node.text]
        if texts:
            paragraphs.append(''.join(texts))

    return ' '.join(paragraphs)

# The beginning of the loop

with open('nifty_next_tickers.pickle', 'rb') as f:
    tickers = pickle.load(f)

count = 0
for ticker in tickers:
    my_dir = f"C:/Users/SKA/OneDrive/Desktop/annual_reports/"

    my_docx = glob.glob(f"{ticker}/*.docx")

    for file_path in my_docx:

        file_name = file_path.split(".")[0].split("\\")[1]

        if os.path.exists(f"{ticker}\\{file_name}.pickle"):
            print(f"Already have {ticker}\\{file_name}.pickle")

        else:
            try:
                text = get_docx_text(file_path)

                with open(f"{ticker}\\{file_name}.pickle", 'wb') as f:
                    pickle.dump(text, f)
                print(f"Converted {ticker}\\{file_name}.docx to {file_name}.pickle")

            except Exception as e:
                print(f"File at {ticker}\\{file_name}.docx cause a problem")
                print(str(e))
    count += 1
    print(count)


from gtts import gTTS
import random
from playsound import playsound
import speech_recognition as sr
import os

class PersonalAssistant:
    def __init__(self, recognizer, microphone, keywordlist):
        self.recognizer = recognizer
        self.microphone = microphone
        self.keywordlist = keywordlist

    def respondHelper(self, keyword):
        prefile = "responsetextfiles/"
        if keyword == "special":
            f = open(prefile + "sassy.txt")
            linenum = random.randint(0, 4)
        elif keyword == "hello":
            f = open(prefile + "greeting.txt")
            linenum = random.randint(0, 11)
        else:
            print("well, the keywords weren't chosen")
            return

        for i, line in enumerate(f):
            if i == linenum:
                return line
            elif i > linenum:
                break
        f.close()

    def listeningNow(self):
        # check that recognizer and microphone arguments are appropriate type
        if not isinstance(self.recognizer, sr.Recognizer):
            raise TypeError("`recognizer` must be `Recognizer` instance")

        if not isinstance(self.microphone, sr.Microphone):
            raise TypeError("`microphone` must be `Microphone` instance")

        # let's have a listen
        with microphone as source:
            self.recognizer.adjust_for_ambient_noise(source)
            audio = self.recognizer.listen(source)
        print("Say something")
        try:
            googleGuess = self.recognizer.recognize_google(audio)
            print("Did you say: " + googleGuess)
        except sr.RequestError:
            # API was unreachable or unresponsive
            print("API unavailable")
            googleGuess = "API unavailable"
        except sr.UnknownValueError:
            # speech was unintelligible
            print("Sorry, I couldn't understand that")
            googleGuess = "Sorry, I couldn't understand that"
        return str(googleGuess)

    def recognize_speech_from_mic(self):
        """Transcribe speech from recorded from `microphone`.

        Returns a dictionary with three keys:
        "success": a boolean indicating whether or not the API request was
                   successful
        "error":   `None` if no error occured, otherwise a string containing
                   an error message if the API could not be reached or
                   speech was unrecognizable
        "transcription": `None` if speech could not be transcribed,
                   otherwise a string containing the transcribed text
        """
        # check that recognizer and microphone arguments are appropriate type
        if not isinstance(recognizer, sr.Recognizer):
            raise TypeError("`recognizer` must be `Recognizer` instance")

        if not isinstance(microphone, sr.Microphone):
            raise TypeError("`microphone` must be `Microphone` instance")

        # adjust the recognizer sensitivity to ambient noise and record audio
        # from the microphone
        with microphone as source:
            recognizer.adjust_for_ambient_noise(source)
            audio = recognizer.listen(source)

        # set up the response object
        response = {
            "success": True,
            "error": None,
            "transcription": None
        }

        # try recognizing the speech in the recording
        # if a RequestError or UnknownValueError exception is caught,
        #     update the response object accordingly
        try:
            response["transcription"] = recognizer.recognize_google(audio)
        except sr.RequestError:
            # API was unreachable or unresponsive
            response["success"] = False
            response["error"] = "API unavailable"
        except sr.UnknownValueError:
            # speech was unintelligible
            response["error"] = "Unable to recognize speech"

        return response

    def respond(self, keyword):
        # here is where we respond
        # we get the phrase from the respondHelper function, passing in the keyword (i.e. "hello")
        phrase = self.respondHelper(keyword)
        tts = gTTS(phrase, 'en')

        # For whatever reason, running the program multiple times
        # with the same response.mp3 file causes it to stop working.
        # It seems changing the file name resolves this issue.
        r1 = str(random.randint(1, 1000000))
        r2 = str(random.randint(1, 1000000))
        randfile = r1 + "response" + r2 + ".mp3"
        tts.save(randfile)
        playsound(randfile)
        # We remember to delete the response file after this program finishes.
        # The deletion probably won't show on the windows explorer window
        # until the program ends (either by error or completion)
        os.remove(randfile)

if __name__ == "__main__":
    # create recognizer and mic instances
    recognizer = sr.Recognizer()
    microphone = sr.Microphone()

    # keywords are special words which we can have special actions for if we hear them
    keywordlist = ["special", "hello"]

    # Lovely name, isn't it?
    cassidy = PersonalAssistant(recognizer, microphone, keywordlist)

    done = False
    while not done:
        for j in range(6):  # we'll loop to keep checking for input
            print("Say something!")
            # here, we call our function to grab our words and transcribe it,
            # passing into "guess" a dictionary of values to work with
            guess = cassidy.recognize_speech_from_mic()
            if guess["transcription"]:
                break
            if not guess["success"]:
                break
            print("I didn't catch that. What did you say?\n")
        # if there was an error, stop break out of loop
        if guess["error"]:
            print("ERROR: {}".format(guess["error"]))
            break

        if guess["transcription"]:  # i.e. if the value of "transcription" != None
            audioTranscribed = str(guess["transcription"])
            print("You said: " + audioTranscribed)
            if audioTranscribed in keywordlist:  # if it's a word in our special keywords list
                for i in range(5):
                    try:
                        cassidy.respond(audioTranscribed)
                        break
                    except:
                        print("response failed, trying again")
                        pass
            elif audioTranscribed == "close program":
                done = True
            # if neither if nor elif, then we'll listen again for more input
            elif "open" in audioTranscribed:
                audioTranscribedList = audioTranscribed.split()
                if audioTranscribedList.__len__() == 2:
                    # I want to search in the command prompt for the applicationName + ".exe" or something
                    pass


import datetime
import json
import requests
from matplotlib import pylab

def get_date(url: str):
    """
    返回json格式数据
    :param url: 获取数据API
    :return: dict
    """
    message = ''
    try:
        url = url
        repo_date = json.loads(requests.get(url).text)
        if repo_date['errMsg'] == "9999":
            store_date = {
                'yesterdayBuyNum': repo_date['data']['yesterdayBuyNum'],
                'yesterdayUseNum': repo_date['data']['yesterdayUseNum'],
                'yesterdayUseAmount': repo_date['data']['yesterdayUseAmount'],
                'totalUseAmount': repo_date['data']['totalUseAmount'],
                'totalBuyNum': repo_date['data']['totalBuyNum'],
                'totalUseNum': repo_date['data']['totalUseNum'],
            }
            return store_date
        else:
            message = repo_date['errMsg']
    except:
        raise Exception(message)

def make_text(date: dict):
    """
    返回拼装数据
    :param yesterdayBuyNum: 当日旅游套票销售数量
    :param yesterdayUseNum: 当日使用套票乘车人数
    :param yesterdayUseAmount: 当日使用套票乘车金额
    :param totalUseAmount: 累计使用套票乘车金额
    :param totalBuyNum: 累计旅游套票销售数量
    :param totalUseNum: 累计使用套票乘车人数
    :return: str
    """
    week = {'1': '一', '2': '二', '3': '三', '4': '四', '5': '五', '6': '六', '0': '日'}
    now_date = datetime.datetime.now().strftime('%Y-%m-%d')
    num_week = datetime.datetime.now().strftime('%w')
    now_week = week[num_week]
    title_text = "x旅游套票信息\n当日旅游套票销售数量: {0}套".format(date['yesterdayBuyNum'])
    content_text = ""
    content_text += "当日旅游套票销售数量: {0}套\n" \
                    "当日使用套票乘车人数: {1}人\n" \
                    "当日使用套票乘车金额: {2}元\n" \
                    "------------------------------\n" \
                    "累计旅游套票销售数量: {3}套\n" \
                    "累计使用套票乘车人数: {4}人\n" \
                    "累计使用套票乘车金额: {5}元\n" \
                    "------------------------------\n" \
                    "信息来源: x\n" \
                    "({6} 星期{7})".format(date['yesterdayBuyNum'], date['yesterdayUseNum'], date['yesterdayUseAmount'],
                                         date['totalUseAmount'], date['totalBuyNum'], date['totalUseNum'], now_date,
                                         now_week)
    return (title_text, content_text)

def change_style(title_text: str, content_text: str):
    """
    处理文本转为图片
    :param title_text: 标题
    :param content_text: 文本
    :return: None
    """
    # 图片名
    n_date = datetime.datetime.now().strftime('%Y%m%d%H%M%S')
    pic_name = 'citypass' + n_date + '.png'
    font_title = {
        'family': 'KaiTi',
        'style': 'italic',
        'weight': 'black',
        'color': 'black',
        'size': 24
    }
    font_text = {
        'family': 'KaiTi',
        'style': 'normal',
        'weight': 'normal',
        'color': 'black',
        'size': 20
    }
    pylab.axis('off')
    pylab.text(0, 0.9, title_text, fontdict=font_title)
    pylab.text(0.05, 0, content_text, fontdict=font_text)
    pylab.savefig(pic_name)
    return pic_name

def send_msg(webhook, message: str):
    News = {
        "msgtype": "markdown",
        "markdown": {
            "title": "x数据",
            "text": ">![screenshot](%s)" % (message)
        }
    }
    headers = {'Content-Type': 'application/json'}
    requests.post(webhook, data=json.dumps(News), headers=headers)

if __name__ == "__main__":
    url = "http://x.x.x.x:x/dingdingData"
    res_title_text, res_text = make_text(get_date(url))
    pic_name = change_style(res_title_text, res_text)
    # 配置nginx图片链接
    message = 'http://x.x.x.x:x/citypass/' + pic_name
    # 钉钉机器人测试
    webhook = "https://oapi.dingtalk.com/robot/send?access_token=x"
    send_msg(webhook=webhook, message=message)


from genericTree.GenericTree import GenericTree
from avlTree.Node import Node

'''
Class to create AVLTree objects
'''
class AVLTree(GenericTree):

    '''
    Constructor for a AVLTree.
    '''
    def __init__(self, *args):
        self.node = None
        self.height = 1
        self.balance = 0;

        #If Input contains only one value it becomes the Root
        if len(args) == 1:
            for i in args[0]:
                self.insert(i)

    '''
    Returns the height of the Tree.
    '''
    def getHeight(self):
        if self.node != None:
            tmpReturn=max(self.node.left.getHeight() , self.node.right.getHeight())+1
        else:
            tmpReturn=0
        return tmpReturn

    '''
    Checks if Root is Leaf.
    '''
    def isLeaf(self):
        return (self.height == 0)

    '''
    Inserts value in the Tree
    '''
    def insert(self, key):
        tree = self.node

        newnode = Node(key)

        if tree == None:
            self.node = newnode
            self.node.left = AVLTree()
            self.node.right = AVLTree()

        #value is smaller then Tree Node
        #value goes left
        elif key < tree.key:
            self.node.left.insert(key)

        #value is bigger then Tree Node
        #value goes right
        elif key > tree.key:
            self.node.right.insert(key)

        #Rebalances Tree after inserting
        self.rebalance()

    '''
    Rebalances the Tree by rotating Left or Right
    '''
    def rebalance(self):
        '''
        Rebalance a particular (sub)tree
        '''
        # value inserted. Let's check if we're balanced
        self.updateHeights(False)
        self.updateBalances(False)
        #While Tree is unbalanced
        while self.balance < -1 or self.balance > 1:
            if self.balance > 1:
                if self.node.left.balance < 0:
                    self.node.left.rotateLeft() # we're in case II
                    self.updateHeights()
                    self.updateBalances()
                self.rotateRight()
                self.updateHeights()
                self.updateBalances()

            if self.balance < -1:
                if self.node.right.balance > 0:
                    self.node.right.rotateRight() # we're in case III
                    self.updateHeights()
                    self.updateBalances()
                self.rotateLeft()
                self.updateHeights()
                self.updateBalances()

    '''
    Rotates unbalanced Nodes to the right
    '''
    def rotateRight(self):
        # Rotate right pivoting on self

        A = self.node
        B = self.node.left.node
        T = B.right.node

        self.node = B
        B.right.node = A
        A.left.node = T

    '''
    Rotates unbalanced Nodes to the Left
    '''
    def rotateLeft(self):
        # Rotate left pivoting on self

        A = self.node
        B = self.node.right.node
        T = B.left.node

        self.node = B
        B.left.node = A
        A.right.node = T

    '''
    checks again the Height after the Balancing
    '''
    def updateHeights(self, recurse=True):
        if not self.node == None:
            if recurse:
                if self.node.left != None:
                    self.node.left.updateHeights()
                if self.node.right != None:
                    self.node.right.updateHeights()

            self.height = max(self.node.left.height,
                              self.node.right.height) + 1
        else:
            self.height = -1

    '''
    checks again the Balance after rebalancing
    '''
    def updateBalances(self, recurse=True):
        if not self.node == None:
            if recurse:
                if self.node.left != None:
                    self.node.left.updateBalances()
                if self.node.right != None:
                    self.node.right.updateBalances()

            self.balance = self.node.left.height - self.node.right.height
        else:
            self.balance = 0

    '''
    Deletes a Node of the Tree
    '''
    def delete(self, key):
        # debug("Trying to delete at node: " + str(self.node.key))
        if self.node != None:
            if self.node.key == key:
                if self.node.left.node == None and self.node.right.node == None:
                    self.node = None # leaves can be killed at will
                # if only one subtree, take that
                elif self.node.left.node == None:
                    self.node = self.node.right.node
                elif self.node.right.node == None:
                    self.node = self.node.left.node

                # worst-case: both children present. Find logical successor
                else:
                    replacement = self.successor(self.node)
                    if replacement != None: # sanity check
                        self.node.key = replacement.key

                        # replaced. Now delete the key from right child
                        self.node.right.delete(replacement.key)

                self.rebalance()
            elif key < self.node.key:
                self.node.left.delete(key)
            elif key > self.node.key:
                self.node.right.delete(key)

            self.rebalance()

    '''
    Searches for the smallest valued node in RIGHT child
    '''
    def successor(self, node):
        '''
        Find the smallest valued node in RIGHT child
        '''
        node = node.right.node
        if node != None: # just a sanity check
            tmpReturn=node
            while node.left != None:

                if node.left.node == None:
                    tmpReturn=node
                else:
                    node = node.left.node
        return tmpReturn

    '''
    Checks if Tree is balanced
    '''
    def isBalanced(self):
        if self == None or self.node == None:
            tmpReturn=True
        else:
            # We always need to make sure we are balanced
            self.updateHeights()
            self.updateBalances()
            tmpReturn= ((abs(self.balance) < 2) and self.node.left.isBalanced() and self.node.right.isBalanced())
        return tmpReturn

    '''
    Prints out the Values of the Nodes in Order
    '''
    def preOrderOutput(self):
        if self.node == None:
            tmpReturn=[]
        else:
            inlist = []
            inlist.append(self.node.key)

            l = self.node.left.preOrderOutput()
            for i in l:
                inlist.append(i)

            l = self.node.right.preOrderOutput()
            for i in l:
                inlist.append(i)
            tmpReturn=inlist
        return tmpReturn

    '''
    Returns true if a Node with the given value exists.
    In other cases false will be returned.
    Parameters: value: the value to search the Nodes for.
    '''
    def has(self, key):
        found = False
        inlist = []
        inlist.append(self.node.key)

        l = self.node.left.preOrderOutput()
        for i in l:
            inlist.append(i)

        l = self.node.right.preOrderOutput()
        for i in l:
            inlist.append(i)

        for i in inlist:
            if i == key:
                found = True
        return found


""" cyclicSequence.py

__author__ = "Albert D. Holmes"
__reference__ = "https://app.codesignal.com/tournaments/texbGoi9JwWPzRtJh/D"

"""

def cyclicSequence(sequence):

    cnt = 0
    for i in range(1, len(sequence)):
        if sequence[i] - sequence[i - 1] < 0:
            cnt += 1
        if sequence[i] - sequence[i - 1] == 0:
            return False

    if sequence[0] > sequence[-1]:
        cnt += 1

    return cnt % 2 == 0