import os,sys
# os.chdir(os.path.dirname(os.path.abspath(__file__)))
# sys.path.insert(1, os.path.join(sys.path[0], '..'))

import requests
import pandas as pd
import json
import random
import datetime,time
import logging
import re
import urllib.parse

mode ='local'

if(mode=='vpn'):
    def nsefetch(payload):
        if (("%26" in payload) or ("%20" in payload)):
            encoded_url = payload
        else:
            encoded_url = urllib.parse.quote(payload, safe=':/?&=')
        payload_var = 'curl -b cookies.txt "' + encoded_url + '"' + curl_headers + ''
        try:
            output = os.popen(payload_var).read()
            output=json.loads(output)
        except ValueError:  # includes simplejson.decoder.JSONDecodeError:
            payload2 = "https://www.nseindia.com"
            output2 = os.popen('curl -c cookies.txt "'+payload2+'"'+curl_headers+'').read()

            output = os.popen(payload_var).read()
            output=json.loads(output)
        return output
if(mode=='local'):
    def nsefetch(payload):
        try: #Try the normal way.
            output = requests.get(payload,headers=headers).json()
        except:
            try: # Follow Praty's hack as mentioned here at https://forum.unofficed.com/t/solution-equity-history-function-doesnt-work/1197
                output = requests.session().get(payload,headers=headers).json()

            except: #Mimick FrontPage Visit
                s=requests.session()

                payload2 = "https://www.nseindia.com"
                output2 = s.get(payload2,headers=headers).json()

                output = s.get(payload,headers=headers).json()
        return output

headers = {
    'Connection': 'keep-alive',
    'Cache-Control': 'max-age=0',
    'DNT': '1',
    'Upgrade-Insecure-Requests': '1',
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.79 Safari/537.36',
    'Sec-Fetch-User': '?1',
    'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9',
    'Sec-Fetch-Site': 'none',
    'Sec-Fetch-Mode': 'navigate',
    'Accept-Encoding': 'gzip, deflate, br',
    'Accept-Language': 'en-US,en;q=0.9,hi;q=0.8',
}

#Curl headers
curl_headers = ''' -H "authority: beta.nseindia.com" -H "cache-control: max-age=0" -H "dnt: 1" -H "upgrade-insecure-requests: 1" -H "user-agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.117 Safari/537.36" -H "sec-fetch-user: ?1" -H "accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9" -H "sec-fetch-site: none" -H "sec-fetch-mode: navigate" -H "accept-encoding: gzip, deflate, br" -H "accept-language: en-US,en;q=0.9,hi;q=0.8" --compressed'''

run_time=datetime.datetime.now()

#Constants
indices = ['NIFTY','FINNIFTY','BANKNIFTY']

def running_status():
    start_now=datetime.datetime.now().replace(hour=9, minute=15, second=0, microsecond=0)
    end_now=datetime.datetime.now().replace(hour=15, minute=30, second=0, microsecond=0)
    return start_now<datetime.datetime.now()<end_now

#Getting FNO Symboles
def fnolist():
    # df = pd.read_csv("https://www1.nseindia.com/content/fo/fo_mktlots.csv")
    # return [x.strip(' ') for x in df.drop(df.index[3]).iloc[:,1].to_list()]

    positions = nsefetch('https://www.nseindia.com/api/equity-stockIndices?index=SECURITIES%20IN%20F%26O')

    nselist=['NIFTY','NIFTYIT','BANKNIFTY']

    i=0
    for x in range(i, len(positions['data'])):
        nselist=nselist+[positions['data'][x]['symbol']]

    return nselist

def nsesymbolpurify(symbol):
    symbol = symbol.replace('&','%26') #URL Parse for Stocks Like M&M Finance
    return symbol

def nse_optionchain_scrapper(symbol):
    symbol = nsesymbolpurify(symbol)
    if any(x in symbol for x in indices):
        payload = nsefetch('https://www.nseindia.com/api/option-chain-indices?symbol='+symbol)
    else:
        payload = nsefetch('https://www.nseindia.com/api/option-chain-equities?symbol='+symbol)
    return payload

def oi_chain_builder(symbol,expiry="latest",oi_mode="full"):

    payload = nse_optionchain_scrapper(symbol)

    if(oi_mode=='compact'):
        col_names = ['CALLS_OI','CALLS_Chng in OI','CALLS_Volume','CALLS_IV','CALLS_LTP','CALLS_Net Chng','Strike Price','PUTS_OI','PUTS_Chng in OI','PUTS_Volume','PUTS_IV','PUTS_LTP','PUTS_Net Chng']
    if(oi_mode=='full'):
        col_names = ['CALLS_Chart','CALLS_OI','CALLS_Chng in OI','CALLS_Volume','CALLS_IV','CALLS_LTP','CALLS_Net Chng','CALLS_Bid Qty','CALLS_Bid Price','CALLS_Ask Price','CALLS_Ask Qty','Strike Price','PUTS_Bid Qty','PUTS_Bid Price','PUTS_Ask Price','PUTS_Ask Qty','PUTS_Net Chng','PUTS_LTP','PUTS_IV','PUTS_Volume','PUTS_Chng in OI','PUTS_OI','PUTS_Chart']
    oi_data = pd.DataFrame(columns = col_names)

    #oi_row = {'CALLS_OI':0, 'CALLS_Chng in OI':0, 'CALLS_Volume':0, 'CALLS_IV':0, 'CALLS_LTP':0, 'CALLS_Net Chng':0, 'Strike Price':0, 'PUTS_OI':0, 'PUTS_Chng in OI':0, 'PUTS_Volume':0, 'PUTS_IV':0, 'PUTS_LTP':0, 'PUTS_Net Chng':0}
    oi_row = {'CALLS_OI':0, 'CALLS_Chng in OI':0, 'CALLS_Volume':0, 'CALLS_IV':0, 'CALLS_LTP':0, 'CALLS_Net Chng':0, 'CALLS_Bid Qty':0,'CALLS_Bid Price':0,'CALLS_Ask Price':0,'CALLS_Ask Qty':0,'Strike Price':0, 'PUTS_OI':0, 'PUTS_Chng in OI':0, 'PUTS_Volume':0, 'PUTS_IV':0, 'PUTS_LTP':0, 'PUTS_Net Chng':0,'PUTS_Bid Qty':0,'PUTS_Bid Price':0,'PUTS_Ask Price':0,'PUTS_Ask Qty':0}
    if(expiry=="latest"):
        expiry = payload['records']['expiryDates'][0]
    m=0
    for m in range(len(payload['records']['data'])):
        if(payload['records']['data'][m]['expiryDate']==expiry):
            if(1>0):
                try:
                    oi_row['CALLS_OI']=payload['records']['data'][m]['CE']['openInterest']
                    oi_row['CALLS_Chng in OI']=payload['records']['data'][m]['CE']['changeinOpenInterest']
                    oi_row['CALLS_Volume']=payload['records']['data'][m]['CE']['totalTradedVolume']
                    oi_row['CALLS_IV']=payload['records']['data'][m]['CE']['impliedVolatility']
                    oi_row['CALLS_LTP']=payload['records']['data'][m]['CE']['lastPrice']
                    oi_row['CALLS_Net Chng']=payload['records']['data'][m]['CE']['change']
                    if(oi_mode=='full'):
                        oi_row['CALLS_Bid Qty']=payload['records']['data'][m]['CE']['bidQty']
                        oi_row['CALLS_Bid Price']=payload['records']['data'][m]['CE']['bidprice']
                        oi_row['CALLS_Ask Price']=payload['records']['data'][m]['CE']['askPrice']
                        oi_row['CALLS_Ask Qty']=payload['records']['data'][m]['CE']['askQty']
                except KeyError:
                    oi_row['CALLS_OI'], oi_row['CALLS_Chng in OI'], oi_row['CALLS_Volume'], oi_row['CALLS_IV'], oi_row['CALLS_LTP'],oi_row['CALLS_Net Chng']=0,0,0,0,0,0
                    if(oi_mode=='full'):
                        oi_row['CALLS_Bid Qty'],oi_row['CALLS_Bid Price'],oi_row['CALLS_Ask Price'],oi_row['CALLS_Ask Qty']=0,0,0,0
                    pass

                oi_row['Strike Price']=payload['records']['data'][m]['strikePrice']

                try:
                    oi_row['PUTS_OI']=payload['records']['data'][m]['PE']['openInterest']
                    oi_row['PUTS_Chng in OI']=payload['records']['data'][m]['PE']['changeinOpenInterest']
                    oi_row['PUTS_Volume']=payload['records']['data'][m]['PE']['totalTradedVolume']
                    oi_row['PUTS_IV']=payload['records']['data'][m]['PE']['impliedVolatility']
                    oi_row['PUTS_LTP']=payload['records']['data'][m]['PE']['lastPrice']
                    oi_row['PUTS_Net Chng']=payload['records']['data'][m]['PE']['change']
                    if(oi_mode=='full'):
                        oi_row['PUTS_Bid Qty']=payload['records']['data'][m]['PE']['bidQty']
                        oi_row['PUTS_Bid Price']=payload['records']['data'][m]['PE']['bidprice']
                        oi_row['PUTS_Ask Price']=payload['records']['data'][m]['PE']['askPrice']
                        oi_row['PUTS_Ask Qty']=payload['records']['data'][m]['PE']['askQty']
                except KeyError:
                    oi_row['PUTS_OI'], oi_row['PUTS_Chng in OI'], oi_row['PUTS_Volume'], oi_row['PUTS_IV'], oi_row['PUTS_LTP'],oi_row['PUTS_Net Chng']=0,0,0,0,0,0
                    if(oi_mode=='full'):
                        oi_row['PUTS_Bid Qty'],oi_row['PUTS_Bid Price'],oi_row['PUTS_Ask Price'],oi_row['PUTS_Ask Qty']=0,0,0,0
            else:
                logging.info(m)

            if(oi_mode=='full'):
                oi_row['CALLS_Chart'],oi_row['PUTS_Chart']=0,0
            #oi_data = oi_data.append(oi_row, ignore_index=True)
            #oi_data = pd.concat([oi_data, oi_row], ignore_index=True)
            oi_data = pd.concat([oi_data, pd.DataFrame([oi_row])], ignore_index=True)

            oi_data['time_stamp']=payload['records']['timestamp']
    return oi_data,float(payload['records']['underlyingValue']),payload['records']['timestamp']

def nse_quote(symbol,section=""):
    #https://forum.unofficed.com/t/nsetools-get-quote-is-not-fetching-delivery-data-and-delivery-can-you-include-this-as-part-of-feature-request/1115/4
    symbol = nsesymbolpurify(symbol)

    if(section==""):
        if any(x in symbol for x in fnolist()):
            payload = nsefetch('https://www.nseindia.com/api/quote-derivative?symbol='+symbol)
        else:
            payload = nsefetch('https://www.nseindia.com/api/quote-equity?symbol='+symbol)
        return payload

    if(section!=""):
        payload = nsefetch('https://www.nseindia.com/api/quote-equity?symbol='+symbol+'&section='+section)
        return payload

def nse_expirydetails(payload,i=0): #Can make problem. Use nse_expirydetails_by_symbol()

    expiry_dates = payload['records']['expiryDates']
    expiry_dates = [datetime.datetime.strptime(date, "%d-%b-%Y").date() for date in expiry_dates]
    expiry_dates = [date.strftime("%d-%b-%Y") for date in expiry_dates if date >= datetime.datetime.now().date()]
    currentExpiry=expiry_dates[i]
    currentExpiry = datetime.datetime.strptime(currentExpiry,'%d-%b-%Y').date()  # converting json datetime to alice datetime
    date_today = run_time.strftime('%Y-%m-%d')  # required to remove hh:mm:ss
    date_today = datetime.datetime.strptime(date_today,'%Y-%m-%d').date()
    dte = (currentExpiry - date_today).days
    return currentExpiry,dte

def pcr(payload,inp='0'):
    ce_oi = 0
    pe_oi = 0
    for i in payload['records']['data']:
        if i['expiryDate'] == payload['records']['expiryDates'][inp]:
            try:
                ce_oi += i['CE']['openInterest']
                pe_oi += i['PE']['openInterest']
            except KeyError:
                pass
    return pe_oi / ce_oi

#forum.unofficed.com/t/unable-to-find-nse-quote-meta-api/702/4
#Refer https://forum.unofficed.com/t/changed-the-nse-quote-ltp-function/1276
def nse_quote_ltp(symbol,expiryDate="latest",optionType="-",strikePrice=0):
  payload = nse_quote(symbol)

  meta = "Options"
  if(optionType=="Fut"): meta = "Futures"
  if(optionType=="PE"):optionType="Put"
  if(optionType=="CE"):optionType="Call"

  if(expiryDate=="latest") or (expiryDate=="next"):

    if(meta=="Futures"):
      selected_key = next((key for key in payload["expiryDatesByInstrument"] if "futures" in key.lower()), None)
    if(meta=="Options"):
      selected_key = next((key for key in payload["expiryDatesByInstrument"] if "options" in key.lower()), None)

    expiry_dates=payload["expiryDatesByInstrument"][selected_key]
    expiry_dates = [datetime.datetime.strptime(date, "%d-%b-%Y").date() for date in expiry_dates]
    expiry_dates = [date.strftime("%d-%b-%Y") for date in expiry_dates if date >= datetime.datetime.now().date()]
    if(expiryDate=="latest"): expiryDate=expiry_dates[0]
    if(expiryDate=="next"): expiryDate=expiry_dates[1]

  if(optionType!="-"):
      for i in payload['stocks']:
        if meta in i['metadata']['instrumentType']:
          #print(i['metadata'])
          if(optionType=="Fut"):
              if(i['metadata']['expiryDate']==expiryDate):
                lastPrice = i['metadata']['lastPrice']

          if((optionType=="Put")or(optionType=="Call")):
              if (i['metadata']["expiryDate"]==expiryDate):
                if (i['metadata']["optionType"]==optionType):
                  if (i['metadata']["strikePrice"]==strikePrice):
                    #print(i['metadata'])
                    lastPrice = i['metadata']['lastPrice']

  if(optionType=="-"):
      lastPrice = payload['underlyingValue']

  return lastPrice

# print(nse_quote_ltp("RELIANCE"))
# print(nse_quote_ltp("RELIANCE","latest","Fut"))
# print(nse_quote_ltp("RELIANCE","next","Fut"))
# print(nse_quote_ltp("BANKNIFTY","latest","PE",32000))
# print(nse_quote_ltp("BANKNIFTY","next","PE",32000))
# print(nse_quote_ltp("BANKNIFTY","10-Jun-2021","PE",32000))
# print(nse_quote_ltp("BANKNIFTY","17-Jun-2021","PE",32000))
# print(nse_quote_ltp("RELIANCE","latest","PE",2300))
# print(nse_quote_ltp("RELIANCE","next","PE",2300))

def nse_quote_meta(symbol,expiryDate="latest",optionType="-",strikePrice=0):
  payload = nse_quote(symbol)
  #https://stackoverflow.com/questions/7961363/removing-duplicates-in-lists
  #https://stackoverflow.com/questions/19199984/sort-a-list-in-python

  #BankNIFTY and NIFTY has weekly options. Using this Jugaad which has primary base of assumption that Reliance will not step out of FNO.
  #forum.unofficed.com/t/unable-to-find-nse-quote-meta-api/702/4
  if((symbol in indices) and (optionType=="Fut")):
    dates = expiry_list("RELIANCE","list")
    if(expiryDate=="latest"): expiryDate=dates[0]
    if(expiryDate=="next"): expiryDate=dates[1]

  if(expiryDate=="latest") or (expiryDate=="next"):
    dates=list(set((payload["expiryDates"])))
    dates.sort(key = lambda date: datetime.datetime.strptime(date, '%d-%b-%Y'))
    if(expiryDate=="latest"): expiryDate=dates[0]
    if(expiryDate=="next"): expiryDate=dates[1]

  meta = "Options"
  if(optionType=="Fut"): meta = "Futures"
  if(optionType=="PE"):optionType="Put"
  if(optionType=="CE"):optionType="Call"

  if(optionType!="-"):
      for i in payload['stocks']:
        if meta in i['metadata']['instrumentType']:
          #print(i['metadata'])
          if(optionType=="Fut"):
              if(i['metadata']['expiryDate']==expiryDate):
                metadata = i['metadata']

          if((optionType=="Put")or(optionType=="Call")):
              if (i['metadata']["expiryDate"]==expiryDate):
                if (i['metadata']["optionType"]==optionType):
                  if (i['metadata']["strikePrice"]==strikePrice):
                    #print(i['metadata'])
                    metadata = i['metadata']

  if(optionType=="-"):
      metadata = i['metadata']

  return metadata

def nse_optionchain_ltp(payload,strikePrice,optionType,inp=0,intent=""):
    expiry_dates = payload['records']['expiryDates']
    expiry_dates = [datetime.datetime.strptime(date, "%d-%b-%Y").date() for date in expiry_dates]
    expiry_dates = [date.strftime("%d-%b-%Y") for date in expiry_dates if date >= datetime.datetime.now().date()]
    expiryDate=expiry_dates[inp]
    for x in range(len(payload['records']['data'])):
      if((payload['records']['data'][x]['strikePrice']==strikePrice) & (payload['records']['data'][x]['expiryDate']==expiryDate)):
          if(intent==""): return payload['records']['data'][x][optionType]['lastPrice']
          if(intent=="sell"): return payload['records']['data'][x][optionType]['bidprice']
          if(intent=="buy"): return payload['records']['data'][x][optionType]['askPrice']

def nse_eq(symbol):
    symbol = nsesymbolpurify(symbol)
    try:
        payload = nsefetch('https://www.nseindia.com/api/quote-equity?symbol='+symbol)
        try:
            if(payload['error']=={}):
                print("Please use nse_fno() function to reduce latency.")
                payload = nsefetch('https://www.nseindia.com/api/quote-derivative?symbol='+symbol)
        except:
            pass
    except KeyError:
        print("Getting Error While Fetching.")
    return payload

def nse_fno(symbol):
    symbol = nsesymbolpurify(symbol)
    try:
        payload = nsefetch('https://www.nseindia.com/api/quote-derivative?symbol='+symbol)
        try:
            if(payload['error']=={}):
                print("Please use nse_eq() function to reduce latency.")
                payload = nsefetch('https://www.nseindia.com/api/quote-equity?symbol='+symbol)
        except KeyError:
            pass
    except KeyError:
        print("Getting Error While Fetching.")
    return payload

def quote_equity(symbol):
    return nse_eq(symbol)

def quote_derivative(symbol):
    return nse_fno(symbol)

def option_chain(symbol):
    return nse_optionchain_scrapper(symbol)

def nse_holidays(type="trading"):
    if(type=="clearing"):
        payload = nsefetch('https://www.nseindia.com/api/holiday-master?type=clearing')
    if(type=="trading"):
        payload = nsefetch('https://www.nseindia.com/api/holiday-master?type=trading')
    return payload

def holiday_master(type="trading"):
    return nse_holidays(type)

def nse_results(index="equities",period="Quarterly"):
    if(index=="equities") or (index=="debt") or (index=="sme"):
        if(period=="Quarterly") or (period=="Annual")or (period=="Half-Yearly")or (period=="Others"):
            payload = nsefetch('https://www.nseindia.com/api/corporates-financial-results?index='+index+'&period='+period)
            return pd.json_normalize(payload)
        else:
            print("Give Correct Period Input")
    else:
        print("Give Correct Index Input")

def nse_events():
    output = nsefetch('https://www.nseindia.com/api/event-calendar')
    return pd.json_normalize(output)

def nse_past_results(symbol):
    symbol = nsesymbolpurify(symbol)
    return nsefetch('https://www.nseindia.com/api/results-comparision?symbol='+symbol)

def expiry_list(symbol,type="list"):
    logging.info("Getting Expiry List of: "+ symbol)

    if(type!="list"):
        payload = nse_optionchain_scrapper(symbol)
        payload = pd.DataFrame({'Date':payload['records']['expiryDates']})
        return payload

    if(type=="list"):
        payload = nse_quote(symbol)
        dates=list(set((payload["expiryDates"])))
        dates.sort(key = lambda date: datetime.datetime.strptime(date, '%d-%b-%Y'))
        return dates

def nse_custom_function_secfno(symbol,attribute="lastPrice"):
    positions = nsefetch('https://www.nseindia.com/api/equity-stockIndices?index=SECURITIES%20IN%20F%26O')
    endp = len(positions['data'])
    for x in range(0, endp):
        if(positions['data'][x]['symbol']==symbol.upper()):
            return positions['data'][x][attribute]

def nse_blockdeal():
    payload = nsefetch('https://nseindia.com/api/block-deal')
    return payload

def nse_marketStatus():
    payload = nsefetch('https://nseindia.com/api/marketStatus')
    return payload

def nse_circular(mode="latest"):
    if(mode=="latest"):
        payload = nsefetch('https://nseindia.com/api/latest-circular')
    else:
        payload = nsefetch('https://www.nseindia.com/api/circulars')
    return payload

def nse_fiidii(mode="pandas"):
    try:
        if(mode=="pandas"):
            return pd.DataFrame(nsefetch('https://www.nseindia.com/api/fiidiiTradeReact'))
        else:
            return nsefetch('https://www.nseindia.com/api/fiidiiTradeReact')
    except:
        logger.info("Pandas is not working for some reason.")
        return nsefetch('https://www.nseindia.com/api/fiidiiTradeReact')

def nsetools_get_quote(symbol):
    payload = nsefetch('https://www.nseindia.com/api/equity-stockIndices?index=SECURITIES%20IN%20F%26O')
    for m in range(len(payload['data'])):
        if(payload['data'][m]['symbol']==symbol.upper()):
            return payload['data'][m]

def nse_index():
    payload = nsefetch('https://iislliveblob.niftyindices.com/jsonfiles/LiveIndicesWatch.json')
    payload = pd.DataFrame(payload["data"])
    return payload

def nse_get_index_list():
    payload = nsefetch('https://iislliveblob.niftyindices.com/jsonfiles/LiveIndicesWatch.json')
    payload = pd.DataFrame(payload["data"])
    return payload["indexName"].tolist()

def nse_get_index_quote(index):
    payload = nsefetch('https://iislliveblob.niftyindices.com/jsonfiles/LiveIndicesWatch.json')
    for m in range(len(payload['data'])):
        if(payload['data'][m]["indexName"] == index.upper()):
            return payload['data'][m]

def nse_get_advances_declines(mode="pandas"):
    try:
        if(mode=="pandas"):
            positions = nsefetch('https://www.nseindia.com/api/equity-stockIndices?index=SECURITIES%20IN%20F%26O')
            return pd.DataFrame(positions['data'])
        else:
            return nsefetch('https://www.nseindia.com/api/equity-stockIndices?index=SECURITIES%20IN%20F%26O')
    except:
        logger.info("Pandas is not working for some reason.")
        return nsefetch('https://www.nseindia.com/api/equity-stockIndices?index=SECURITIES%20IN%20F%26O')

def nse_get_top_losers():
    positions = nsefetch('https://www.nseindia.com/api/equity-stockIndices?index=SECURITIES%20IN%20F%26O')
    df = pd.DataFrame(positions['data'])
    df = df.sort_values(by="pChange")
    return df.head(5)

def nse_get_top_gainers():
    positions = nsefetch('https://www.nseindia.com/api/equity-stockIndices?index=SECURITIES%20IN%20F%26O')
    df = pd.DataFrame(positions['data'])
    df = df.sort_values(by="pChange" , ascending = False)
    return df.head(5)

def nse_get_fno_lot_sizes(symbol="all",mode="list"):
    url="https://archives.nseindia.com/content/fo/fo_mktlots.csv"

    if(mode=="list"):
        s=requests.get(url).text
        res_dict = {}
        for line in s.split('\n'):
          if line != '' and re.search(',', line) and (line.casefold().find('symbol') == -1):
              (code, name) = [x.strip() for x in line.split(',')[1:3]]
              res_dict[code] = int(name)
        if(symbol=="all"):
            return res_dict
        if(symbol!=""):
            return res_dict[symbol.upper()]

    if(mode=="pandas"):
        payload = pd.read_csv(url)
        if(symbol=="all"):
            return payload
        else:
            payload = payload[(payload.iloc[:, 1] == symbol.upper())]
            return payload

def whoistheboss():
    return "subhash"

def indiavix():
    payload = nsefetch("https://www.nseindia.com/api/allIndices")
    for x in range(0, len(payload["data"])):
        if(payload["data"][x]["index"]=="INDIA VIX"):
            return payload["data"][x]["last"]

def index_info(index):
    payload = nsefetch("https://www.nseindia.com/api/allIndices")
    for x in range(0, len(payload["data"])):
        if(payload["data"][x]["index"]==index):
            return payload["data"][x]

import math
from scipy.stats import norm

def black_scholes_dexter(S0,X,t,σ="",r=10,q=0.0,td=365):

  if(σ==""):σ =indiavix()

  S0,X,σ,r,q,t = float(S0),float(X),float(σ/100),float(r/100),float(q/100),float(t/td)
  #https://unofficed.com/black-scholes-model-options-calculator-google-sheet/

  d1 = (math.log(S0/X)+(r-q+0.5*σ**2)*t)/(σ*math.sqrt(t))
  #stackoverflow.com/questions/34258537/python-typeerror-unsupported-operand-types-for-float-and-int

  #stackoverflow.com/questions/809362/how-to-calculate-cumulative-normal-distribution
  Nd1 = (math.exp((-d1**2)/2))/math.sqrt(2*math.pi)
  d2 = d1-σ*math.sqrt(t)
  Nd2 = norm.cdf(d2)
  call_theta =(-((S0*σ*math.exp(-q*t))/(2*math.sqrt(t))*(1/(math.sqrt(2*math.pi)))*math.exp(-(d1*d1)/2))-(r*X*math.exp(-r*t)*norm.cdf(d2))+(q*math.exp(-q*t)*S0*norm.cdf(d1)))/td
  put_theta =(-((S0*σ*math.exp(-q*t))/(2*math.sqrt(t))*(1/(math.sqrt(2*math.pi)))*math.exp(-(d1*d1)/2))+(r*X*math.exp(-r*t)*norm.cdf(-d2))-(q*math.exp(-q*t)*S0*norm.cdf(-d1)))/td
  call_premium =math.exp(-q*t)*S0*norm.cdf(d1)-X*math.exp(-r*t)*norm.cdf(d1-σ*math.sqrt(t))
  put_premium =X*math.exp(-r*t)*norm.cdf(-d2)-math.exp(-q*t)*S0*norm.cdf(-d1)
  call_delta =math.exp(-q*t)*norm.cdf(d1)
  put_delta =math.exp(-q*t)*(norm.cdf(d1)-1)
  gamma =(math.exp(-r*t)/(S0*σ*math.sqrt(t)))*(1/(math.sqrt(2*math.pi)))*math.exp(-(d1*d1)/2)
  vega = ((1/100)*S0*math.exp(-r*t)*math.sqrt(t))*(1/(math.sqrt(2*math.pi))*math.exp(-(d1*d1)/2))
  call_rho =(1/100)*X*t*math.exp(-r*t)*norm.cdf(d2)
  put_rho =(-1/100)*X*t*math.exp(-r*t)*norm.cdf(-d2)

  return call_theta,put_theta,call_premium,put_premium,call_delta,put_delta,gamma,vega,call_rho,put_rho

def equity_history_virgin(symbol,series,start_date,end_date):
    #url="https://www.nseindia.com/api/historical/cm/equity?symbol="+symbol+"&series=[%22"+series+"%22]&from="+str(start_date)+"&to="+str(end_date)+""
    url = 'https://www.nseindia.com/api/historical/cm/equity?symbol=' + symbol + '&series=["' + series + '"]&from=' + start_date + '&to=' + end_date

    payload = nsefetch(url)
    return pd.DataFrame.from_records(payload["data"])

# You shall see beautiful use the logger function.
def equity_history(symbol,series,start_date,end_date):
    #We are getting the input in text. So it is being converted to Datetime object from String.
    start_date = datetime.datetime.strptime(start_date, "%d-%m-%Y")
    end_date = datetime.datetime.strptime(end_date, "%d-%m-%Y")
    logging.info("Starting Date: "+str(start_date))
    logging.info("Ending Date: "+str(end_date))

    #We are calculating the difference between the days
    diff = end_date-start_date
    logging.info("Total Number of Days: "+str(diff.days))
    logging.info("Total FOR Loops in the program: "+str(int(diff.days/40)))
    logging.info("Remainder Loop: " + str(diff.days-(int(diff.days/40)*40)))

    total=pd.DataFrame()
    for i in range (0,int(diff.days/40)):

        temp_date = (start_date+datetime.timedelta(days=(40))).strftime("%d-%m-%Y")
        start_date = datetime.datetime.strftime(start_date, "%d-%m-%Y")

        logging.info("Loop = "+str(i))
        logging.info("====")
        logging.info("Starting Date: "+str(start_date))
        logging.info("Ending Date: "+str(temp_date))
        logging.info("====")

        #total=total.append(equity_history_virgin(symbol,series,start_date,temp_date))
        #total=total.concat(equity_history_virgin(symbol,series,start_date,temp_date))
        total = pd.concat([total, equity_history_virgin(symbol, series, start_date, temp_date)])

        logging.info("Length of the Table: "+ str(len(total)))

        #Preparation for the next loop
        start_date = datetime.datetime.strptime(temp_date, "%d-%m-%Y")

    start_date = datetime.datetime.strftime(start_date, "%d-%m-%Y")
    end_date = datetime.datetime.strftime(end_date, "%d-%m-%Y")

    logging.info("End Loop")
    logging.info("====")
    logging.info("Starting Date: "+str(start_date))
    logging.info("Ending Date: "+str(end_date))
    logging.info("====")

    #total=total.append(equity_history_virgin(symbol,series,start_date,end_date))
    #total=total.concat(equity_history_virgin(symbol,series,start_date,end_date))
    total = pd.concat([total, equity_history_virgin(symbol, series, start_date, end_date)])

    logging.info("Finale")
    logging.info("Length of the Total Dataset: "+ str(len(total)))
    payload = total.iloc[::-1].reset_index(drop=True)
    return payload

def derivative_history_virgin(symbol,start_date,end_date,instrumentType,expiry_date,strikePrice="",optionType=""):

    instrumentType = instrumentType.lower()

    if(instrumentType=="options"):
        if("NIFTY" in symbol): instrumentType="FUTSTK"
        instrumentType="OPTSTK"
    if(instrumentType=="futures"):
        if("NIFTY" in symbol): instrumentType="OPTIDX"
        instrumentType="FUTIDX"

    if(((instrumentType=="OPTIDX")or (instrumentType=="OPTSTK")) and (expiry_date!="")):
        strikePrice = "%.2f" % strikePrice
        strikePrice = str(strikePrice)

    nsefetch_url = "https://www.nseindia.com/api/historical/fo/derivatives?&from="+str(start_date)+"&to="+str(end_date)+"&optionType="+optionType+"&strikePrice="+strikePrice+"&expiryDate="+expiry_date+"&instrumentType="+instrumentType+"&symbol="+symbol+""
    payload = nsefetch(nsefetch_url)
    logging.info(nsefetch_url)
    logging.info(payload)
    return pd.DataFrame.from_records(payload["data"])

def derivative_history(symbol,start_date,end_date,instrumentType,expiry_date,strikePrice="",optionType=""):
    #We are getting the input in text. So it is being converted to Datetime object from String.
    start_date = datetime.datetime.strptime(start_date, "%d-%m-%Y")
    end_date = datetime.datetime.strptime(end_date, "%d-%m-%Y")
    logging.info("Starting Date: "+str(start_date))
    logging.info("Ending Date: "+str(end_date))

    #We are calculating the difference between the days
    diff = end_date-start_date
    logging.info("Total Number of Days: "+str(diff.days))
    logging.info("Total FOR Loops in the program: "+str(int(diff.days/40)))
    logging.info("Remainder Loop: " + str(diff.days-(int(diff.days/40)*40)))

    total=pd.DataFrame()
    for i in range (0,int(diff.days/40)):

        temp_date = (start_date+datetime.timedelta(days=(40))).strftime("%d-%m-%Y")
        start_date = datetime.datetime.strftime(start_date, "%d-%m-%Y")

        logging.info("Loop = "+str(i))
        logging.info("====")
        logging.info("Starting Date: "+str(start_date))
        logging.info("Ending Date: "+str(temp_date))
        logging.info("====")

        #total=total.append(derivative_history_virgin(symbol,start_date,temp_date,instrumentType,expiry_date,strikePrice,optionType))
        #total=total.concat([total, derivative_history_virgin(symbol,start_date,temp_date,instrumentType,expiry_date,strikePrice,optionType)])
        total = pd.concat([total, derivative_history_virgin(symbol, start_date, temp_date, instrumentType, expiry_date, strikePrice, optionType)])

        logging.info("Length of the Table: "+ str(len(total)))

        #Preparation for the next loop
        start_date = datetime.datetime.strptime(temp_date, "%d-%m-%Y")

    start_date = datetime.datetime.strftime(start_date, "%d-%m-%Y")
    end_date = datetime.datetime.strftime(end_date, "%d-%m-%Y")

    logging.info("End Loop")
    logging.info("====")
    logging.info("Starting Date: "+str(start_date))
    logging.info("Ending Date: "+str(end_date))
    logging.info("====")

    #total=total.append(derivative_history_virgin(symbol,start_date,end_date,instrumentType,expiry_date,strikePrice,optionType))
    #total = total.concat([total, derivative_history_virgin(symbol,start_date,end_date,instrumentType,expiry_date,strikePrice,optionType)])
    total = pd.concat([total, derivative_history_virgin(symbol, start_date, end_date, instrumentType, expiry_date, strikePrice, optionType)])

    logging.info("Finale")
    logging.info("Length of the Total Dataset: "+ str(len(total)))
    payload = total.iloc[::-1].reset_index(drop=True)
    return payload

def expiry_history(symbol,start_date="",end_date=""):
    if(end_date==""):end_date=end_date
    nsefetch_url = "https://www.nseindia.com/api/historical/fo/derivatives/meta?&from="+start_date+"&to="+end_date+"&symbol="+symbol+""
    payload = nsefetch(nsefetch_url)
    return payload['data'][2]

# # Nifty Indicies Site

niftyindices_headers = {
    'Connection': 'keep-alive',
    'sec-ch-ua': '" Not;A Brand";v="99", "Google Chrome";v="91", "Chromium";v="91"',
    'Accept': 'application/json, text/javascript, */*; q=0.01',
    'DNT': '1',
    'X-Requested-With': 'XMLHttpRequest',
    'sec-ch-ua-mobile': '?0',
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.77 Safari/537.36',
    'Content-Type': 'application/json; charset=UTF-8',
    'Origin': 'https://niftyindices.com',
    'Sec-Fetch-Site': 'same-origin',
    'Sec-Fetch-Mode': 'cors',
    'Sec-Fetch-Dest': 'empty',
    'Referer': 'https://niftyindices.com/reports/historical-data',
    'Accept-Language': 'en-US,en;q=0.9,hi;q=0.8',
}

def index_history(symbol,start_date,end_date):
    data = "{'name':'"+symbol+"','startDate':'"+start_date+"','endDate':'"+end_date+"'}"
    payload = requests.post('https://niftyindices.com/Backpage.aspx/getHistoricaldatatabletoString', headers=niftyindices_headers,  data=data).json()
    payload = json.loads(payload["d"])
    payload=pd.DataFrame.from_records(payload)
    return payload

def index_pe_pb_div(symbol,start_date,end_date):
    data = "{'name':'"+symbol+"','startDate':'"+start_date+"','endDate':'"+end_date+"'}"
    payload = requests.post('https://niftyindices.com/Backpage.aspx/getpepbHistoricaldataDBtoString', headers=niftyindices_headers,  data=data).json()
    payload = json.loads(payload["d"])
    payload=pd.DataFrame.from_records(payload)
    return payload

def index_total_returns(symbol,start_date,end_date):
    data = "{'name':'"+symbol+"','startDate':'"+start_date+"','endDate':'"+end_date+"'}"
    payload = requests.post('https://niftyindices.com/Backpage.aspx/getTotalReturnIndexString', headers=niftyindices_headers,  data=data).json()
    payload = json.loads(payload["d"])
    payload=pd.DataFrame.from_records(payload)
    return payload

def get_bhavcopy(date):
    date = date.replace("-","")
    payload=pd.read_csv("https://archives.nseindia.com/products/content/sec_bhavdata_full_"+date+".csv")
    return payload

def get_bulkdeals():
    payload=pd.read_csv("https://archives.nseindia.com/content/equities/bulk.csv")
    return payload

def get_blockdeals():
    payload=pd.read_csv("https://archives.nseindia.com/content/equities/block.csv")
    return payload

#Request from subhash
## https://unofficed.com/how-to-find-the-beta-of-indian-stocks-using-python/
def get_beta_df_maker(symbol,days):
    if("NIFTY" in symbol):
        end_date = datetime.datetime.now().strftime("%d-%b-%Y")
        end_date = str(end_date)

        start_date = (datetime.datetime.now()- datetime.timedelta(days=days)).strftime("%d-%b-%Y")
        start_date = str(start_date)

        df2=index_history(symbol,start_date,end_date)
        df2["daily_change"]=df2["CLOSE"].astype(float).pct_change()
        df2=df2[['HistoricalDate','daily_change']]
        df2 = df2.iloc[1: , :]
        return df2
    else:
        end_date = datetime.datetime.now().strftime("%d-%m-%Y")
        end_date = str(end_date)

        start_date = (datetime.datetime.now()- datetime.timedelta(days=days)).strftime("%d-%m-%Y")
        start_date = str(start_date)

        df = equity_history(symbol,"EQ",start_date,end_date)

        df["daily_change"]=df["CH_CLOSING_PRICE"].pct_change()
        df=df[['CH_TIMESTAMP','daily_change']]
        df = df.iloc[1: , :] #thispointer.com/drop-first-row-of-pandas-dataframe-3-ways/
        return df

def getbeta(symbol,days=365,symbol2="NIFTY 50"):
    return get_beta(symbol,days,symbol2)

def get_beta(symbol,days=365,symbol2="NIFTY 50"):
    #Default is 248 days. (Input of Subhash)
    df = get_beta_df_maker(symbol,days)
    df2 = get_beta_df_maker(symbol2,days)

    x=df["daily_change"].tolist()
    y=df2["daily_change"].tolist()
    #stackoverflow.com/questions/42670055/is-there-any-better-way-to-calculate-the-covariance-of-two-lists-than-this
    mean_x = sum(x) / len(x)
    mean_y = sum(y) / len(y)
    covariance = sum((a - mean_x) * (b - mean_y) for (a,b) in zip(x,y)) / len(x)

    mean = sum(y) / len(y)
    variance = sum((i - mean) ** 2 for i in y) / len(y)

    beta = covariance/variance
    return round(beta,3)

def nse_preopen(key="NIFTY",type="pandas"):
    payload = nsefetch("https://www.nseindia.com/api/market-data-pre-open?key="+key+"")
    if(type=="pandas"):
        payload = pd.DataFrame(payload['data'])
        payload  = pd.json_normalize(payload['metadata'])
        return payload
    else:
        return payload

#By Avinash https://forum.unofficed.com/t/nsepython-documentation/376/102?u=dexter
def nse_preopen_movers(key="FO",filter=1.5):
    preOpen_gainer=nse_preopen(key)
    return preOpen_gainer[preOpen_gainer['pChange'] >1.5],preOpen_gainer[preOpen_gainer['pChange'] <-1.5]

# type = "securities"
# type = "etf"
# type = "sme"
#
# sort = "volume"
# sort = "value"

def nse_most_active(type="securities",sort="value"):
    payload = nsefetch("https://www.nseindia.com/api/live-analysis-most-active-"+type+"?index="+sort+"")
    payload = pd.DataFrame(payload["data"])
    return payload

def nse_eq_symbols():
    #https://forum.unofficed.com/t/feature-request-stocklist-api/1073/11
    eq_list_pd = pd.read_csv('https://archives.nseindia.com/content/equities/EQUITY_L.csv')
    return eq_list_pd['SYMBOL'].tolist()

def nse_price_band_hitters(bandtype="both",view="AllSec"):
  payload = nsefetch("https://www.nseindia.com/api/live-analysis-price-band-hitter")

  #bandtype can be upper, lower, both
  #view can be AllSec,SecGtr20,SecLwr20
  return pd.DataFrame(payload[bandtype][view]["data"])

def nse_largedeals(mode="bulk_deals"):
  payload = nsefetch('https://www.nseindia.com/api/snapshot-capital-market-largedeal')
  if(mode=="bulk_deals"):
    return pd.DataFrame(payload["BULK_DEALS_DATA"])
  if(mode=="short_deals"):
    return pd.DataFrame(payload["SHORT_DEALS_DATA"])
  if(mode=="block_deals"):
    return pd.DataFrame(payload["BLOCK_DEALS_DATA"])

def nse_largedeals_historical(from_date, to_date, mode="bulk_deals"):
    if mode == "bulk_deals":
        mode = "bulk-deals"
    elif mode == "short_deals":
        mode = "short-selling"
    elif mode == "block_deals":
        mode = "block-deals"

    url='https://www.nseindia.com/api/historical/' + mode + '?from=' + from_date + '&to=' + to_date
    logging.info("Fetching " + str(url))
    payload = nsefetch(url)
    return pd.DataFrame(payload["data"])

#https://forum.unofficed.com/t/feature-request-nse-fno-participant-wise-oi/1179/7
#print(get_fao_participant_oi("04-06-2021"))
def get_fao_participant_oi(date):
    date = date.replace("-","")
    payload=pd.read_csv("https://archives.nseindia.com/content/nsccl/fao_participant_oi_"+date+".csv")
    return payload

#https://forum.unofficed.com/t/how-to-check-if-the-market-is-open-today-or-not/1268/1
def is_market_open(segment = "FO"): #COM,CD,CB,CMOT,COM,FO,IRD,MF,NDM,NTRP,SLBS

    holiday_json = nse_holidays()[segment]

    # Get today's date in the format 'dd-Mon-yyyy'
    today_date = datetime.date.today().strftime('%d-%b-%Y')

    # Check if today's date is in the holiday_json
    for holiday in holiday_json:
        if holiday['tradingDate'] != today_date:
            print("FNO Market is open today. Have a Nice Trade!")
            return True
        if holiday['tradingDate'] == today_date:
            print(f"Market is closed today because of {holiday['description']}")
            return False

def nse_expirydetails_by_symbol(symbol,meta ="Futures",i=0):
    payload = nse_quote(symbol)

    if(meta=="Futures"):
      selected_key = next((key for key in payload["expiryDatesByInstrument"] if "futures" in key.lower()), None)
    if(meta=="Options"):
      selected_key = next((key for key in payload["expiryDatesByInstrument"] if "options" in key.lower()), None)

    expiry_dates=payload["expiryDatesByInstrument"][selected_key]
    expiry_dates = [datetime.datetime.strptime(date, "%d-%b-%Y").date() for date in expiry_dates]
    expiry_dates = [date.strftime("%d-%b-%Y") for date in expiry_dates if date >= datetime.datetime.now().date()]

    currentExpiry=expiry_dates[i]
    currentExpiry = datetime.datetime.strptime(currentExpiry,'%d-%b-%Y').date()
    dte = (currentExpiry - datetime.datetime.now().date()).days
    return currentExpiry,dte


import os
if "rrszynka" not in os.getcwd():
    raise Exception("Working locally - is this desired?")
import os, shutil
import bb_utils
import bb_utils.meta
import bb_utils.ids
import bb_backend
import datetime
import time
import psycopg2
import psycopg2.extras
import numpy as np
import pandas as pd
from tqdm import tqdm
from pathlib import Path
import datetime
from datetime import timedelta
import sys
#%%

# bb_backend.api.server_adress = 'localhost:8000'
connect_str = """dbname='beesbook' user='reader' host='tonic.imp.fu-berlin.de' password='' application_name='mehmed'"""
#removed storage from filepath

#TODO: defined here, main notebook and file_helpers - how to extract it to be just in one place?
cache_location_prefix = "/home/mi/rrszynka/mnt/janek/"+"caches/" # "/home/mi/rrszynka/mnt/janek"+"caches/" for mnt
detections_cache_path = cache_location_prefix + "Detections/"
alternate_detections_cache_path = '/home/mi/rrszynka/caches/Detections/'

def cache_detections_from_database(datetime_start, observ_period=datetime.timedelta(hours=1), num_observ_periods=24):
    """Pulls detections from the Beesbook database
    and saves them to disk as (e.g.) "DETECTIONS-2016-08-25_conf_07.csv" file.

    Args:
        datetime_start (datetime): the starting point for observations we want to pull
        observ_period (timedelta):  size of the window we're going to save to disk as a single file
        num_observ_periods (int):  how many periods (files) we want to save

    Deprecated arg:
        detection_confidence_requirement (float): how high the database value for detection confidence needs to be.
        As of 17.01, this is always set to 0 and is meant to be configured at a later stage (detections_to_presence)
    """

    print('Beginning at ' + datetime_start.strftime("%Y-%m-%d %H:%M:%S"))
    print('Saving to ' + detections_cache_path)

    for i in tqdm(range(0,num_observ_periods)):
        datetime_end = datetime_start + observ_period
        datetime_str = datetime_start.strftime("%Y-%m-%d_%H:%M:%S")
        #If this file has already been saved, we're good, just drop it

        filename = 'DETECTIONS-'+datetime_str+'.csv'
        filepath = detections_cache_path+filename
        file = Path(filepath)
        if file.exists():
            print("File " + filename + " already exists, dropping")
            datetime_start = datetime_end
            continue
        start = time.time()
        with psycopg2.connect(connect_str) as conn:
            query = """SET geqo_effort to 10;
                        SET max_parallel_workers_per_gather TO 8;
                        SET temp_buffers to "32GB";
                        SET work_mem to "1GB";
                        SET temp_tablespaces to "ssdspace";
                    SELECT * FROM bb_detections_2016_stitched
                   WHERE timestamp >= %s AND
                         timestamp < %s
                   ;"""
        df = pd.read_sql_query(
            query, conn,
            params=(datetime_start, datetime_end),
            coerce_float=False)

        df.to_csv(filepath)
        datetime_start = datetime_end

# cache_detections_from_database(datetime.datetime(2016,7,20)) #TEST
#2.68 mins per hour of detections at midnight

#DEPRECATED
def create_presence_cache_filename(datetime_start,
                                   num_hours=24,
                                   num_intervals_per_hour=120,
                                   cams=[1,2,3,4],
                                   method='binary',
                                   detection_confidence_requirement=0):

    presence_cache_location_prefix = cache_location_prefix + "Presence/"

    date_string = (datetime_start).strftime("%Y-%m-%d_%H")
    conf_string = str(detection_confidence_requirement).replace('.','')
    cams_string = ''.join(str(x) for x in cams)
    csv_name = 'PRESENCE-'+method+'-'+str(date_string)+"_num_hours_"+str(num_hours)+"_int_size_"+str(num_intervals_per_hour)+'_conf_'+conf_string+'_cams_'+cams_string+'.csv'
    csv_path = presence_cache_location_prefix+csv_name
    return (csv_name, csv_path)

def create_gaps_cache_filename(date,
                               detection_confidence_requirement=0):
    gaps_cache_location_prefix = cache_location_prefix + "Gaps/"
    conf_string = str(detection_confidence_requirement).replace('.','')
    csv_name = 'GAPS-'+str(date)+'_conf_'+conf_string+'.pkl'
    csv_path = gaps_cache_location_prefix+csv_name
    return (csv_name, csv_path)

def create_detections_cache_filename(datetime_start, interval_size=datetime.timedelta(hours=1), alternate_location=False):
    date_string = (datetime_start).strftime('%Y-%m-%d_%H:%M:%S')
    csv_name = "DETECTIONS-" + date_string + '.csv'
    csv_path = detections_cache_path+csv_name
    if alternate_location == True:
        csv_path = alternate_detections_cache_path + csv_name
    return (csv_name, csv_path)

def detections_to_presence(num_hours, datetime_start, num_intervals_per_hour, bee_ids, cams=[0,1,2,3], method='binary', detection_confidence_requirement=0, return_mode='data'):
    """Reads a number of DETECTION csvs, combines them into one and converts to a measurement of presence guided by the specified method
       and filtered by specified cam numbers. Saves the PRESENCE cache and returns the path to the saved file OR the data itself (depending on return_mode)

    Output: A PRESENCE dataframe (csv file), of size [bee_ids x total_num_intervals]. Contents depend on method (see below in Args).

    Args:
        num_hours(int): number of hours to be processed, i.e. number of hour-long DETECTION csv files to be loaded and operated on
        datetime_start (datetime): the starting point of the observations we want to pull
        num_intervals_per_hour (int):  number of intervals per hour. e.g. a value 120 means that one interval covers 30secs of real time
        bee_ids ([int]):  list of bee ids to be taken into account
        cams ([int]): list of desired cams; all cams = [0,1,2,3]; front cams = [0,1]; back cams = [2,3]
        method (string): 'binary' will return 0 and 1, 1 if the given bee was detected at least once in the given interal
                         'counts' will return a value from 0 to MAX, representing how many times the bee was detected during the given interval
        detection_confidence_requirement (float): how high the database value for detection confidence needs to be for this function to include that detection
        return_mode: 'path' or 'data'
    """
    print('HI')
    if method not in ['binary', 'counts', 'last-location', 'mean-location']:
        print('Please specify a method.')
        return None

    # 1. Prepare paths and filenames
    (csv_name, csv_path) = create_presence_cache_filename(datetime_start,
                                        num_hours,
                                        num_intervals_per_hour,
                                        cams=cams,
                                        method=method,
                                        detection_confidence_requirement=detection_confidence_requirement)

    conf_string = str(detection_confidence_requirement).replace('.','')

    if os.path.isfile(csv_path):
        print('File ' + csv_name + ' already exists, exiting with no effect.')
        return

    # 2.Read and concat a given number of hour-long csvs (note: this was done hour-by-hour because thekla memory would crash if attempting >16h at a time)
    detections_dfs = []

    files = []
    # First check if all files exist
    for i in range(0, num_hours):
        (csv_name, csv_pathname) = create_detections_cache_filename(datetime_start + datetime.timedelta(hours=i))
        if os.path.isfile(csv_pathname) == False:
            # Try the other location
            (csv_name, csv_pathname) = create_detections_cache_filename(datetime_start + datetime.timedelta(hours=i), alternate_location=True)
            print("not found on mnt, trying now locally: " + csv_pathname)
            if os.path.isfile(csv_pathname) == False:
                raise ValueError('File missing: ' + csv_pathname + ' (and potentially more)')
            files.append((csv_name, csv_pathname))

    # Then read and concat them
    for i in tqdm(range(0, num_hours)):
        (csv_name, csv_pathname) = files[i]
        detections_1h = pd.read_csv(csv_pathname,
                                    parse_dates=['timestamp'],
                                    usecols=['timestamp', 'bee_id', 'x_pos_hive', 'y_pos_hive', 'orientation', 'cam_id', 'bee_id_confidence'])
        # 2a. Filter detections to only come from desired cams
        detections_1h = detections_1h[detections_1h.cam_id.isin(cams)]
        detections_dfs.append(detections_1h)
    detections_df = pd.concat(detections_dfs)

    # 2b. Filter detections to be over a certain confidence threshold
    rows_total = detections_df.shape[0]

    print('Num. rows after concatenating: ' + str(rows_total))
    fout = detections_df[detections_df['bee_id_confidence'] < detection_confidence_requirement]
    print('dreq = ' + str(detection_confidence_requirement) +' Will filter out ' + str(fout.shape[0]) + ' rows')
    detections_df = detections_df[detections_df['bee_id_confidence'] > detection_confidence_requirement]

    print('Num. rows after confidence filtering : ' + str(detections_df.shape[0]) + " (percentage: " + str(detections_df.shape[0]/rows_total) + ")")

    #3. Prepare a zeroes dataframe with presence intervals, to be filled up by information from detections_df
    # prepare shape [num_bees x num_intervals]
    presence_df = pd.DataFrame(data=np.zeros([len(bee_ids),(num_intervals_per_hour*num_hours)]))
    presence_df.index = bee_ids

    # 4. Iterate over intervals and over detections to fill up infotmation
    # using user's chosen method (binary/counts/last-location)
    total_num_intervals = num_intervals_per_hour*num_hours
    interval_length = datetime.timedelta(hours=num_hours) // total_num_intervals
    interval_starttime = datetime_start

    for interval in tqdm(range(total_num_intervals)):
        #choose detections for interval
        interval_endtime = interval_starttime + interval_length
        after_start = detections_df['timestamp'] >= interval_starttime
        before_end = detections_df['timestamp'] < interval_endtime
        interval_detections = detections_df[after_start & before_end].fillna(0)

        if method == 'binary':
            # if bee id is detected at least once in this interval, put 1; otherwise put 0
            bee_row_number = 0
            for bee in presence_df.index:
                if bee in interval_detections['bee_id'].unique():
                    presence_df.set_value(bee_row_number, interval, 1)
                bee_row_number += 1

        elif method == 'counts':
            # save the number of times a given bee_id was detected in this interval
            counts = interval_detections['bee_id'].value_counts()
            keys = counts.keys().tolist()
            counts = counts.tolist()

            for i in np.arange(0,len(counts)):
                bee = keys[i]
                # If the bee id had some detections, but wasn't among the bees marked as alive for that day, we ignore it
                if bee in bee_ids:
                    presence_df.at[bee, interval] = counts[i]

        elif method == 'last-location':
            bee_row_number = 0
            for bee in presence_df.index:
                if bee in interval_detections['bee_id'].unique():
                    #TODO: currently just taking the coordinate of the last detection in the interval, maybe change to the average of the interval later?
                    x_c = interval_detections['x_pos_hive'][(interval_detections['bee_id'] == bee)].tail(1).values.item()
                    y_c = interval_detections['y_pos_hive'][(interval_detections['bee_id'] == bee)].tail(1).values.item()
                    coordinates = (round(x_c), round(y_c))
                    presence_df[interval] = presence_df[interval].astype(object) # Generalize type of the cells to handle (x,y) coordinate tuples
                    presence_df.set_value(bee_row_number, interval, coordinates) # Fill in the cell
                bee_row_number += 1

        elif method == 'mean-location':
            for bee in presence_df.index:
                if bee in interval_detections['bee_id'].unique():
                    #TODO: currently just taking the coordinate of the last detection in the interval, maybe change to the average of the interval later?
                    x_c = interval_detections['x_pos_hive'][(interval_detections['bee_id'] == bee)].mean()
                    y_c = interval_detections['y_pos_hive'][(interval_detections['bee_id'] == bee)].mean()
                    coordinates = (round(x_c), round(y_c))
                    presence_df[interval] = presence_df[interval].astype(object) # Generalize type of the cells to handle (x,y) coordinate tuples
                    presence_df.at[bee, interval] = coordinates # Fill in the cell

        interval_starttime = interval_endtime

    presence_df.to_csv(csv_path)
    print("SAVED", csv_path)
    if return_mode == 'path':
        return csv_path
    elif return_mode == 'data':
        return presence_df

def delete_detection_caches_for_date(date_string, directory=detections_cache_path):
    # date_string must be in MM-DD_hh:mm:ss format
    name_prefix = "DETECTIONS-"
    removed_count = 0
    for the_file in os.listdir(detections_cache_path):
        if the_file.startswith(name_prefix+date_string):
            file_path = os.path.join(detections_cache_path, the_file)
            try:
                if os.path.isfile(file_path):
                    print("Removing " + the_file)
                    os.unlink(file_path)
            except Exception as e:
                print(e)

def cache_death_dates():
    """Pulls alive_bees_2016 from the Beesbook database
    and saves a dataframe with the last date alive for each bee to disk as "Last_day_alive.csv" file.
    """

    print('Loading bee alive states')

    with psycopg2.connect(connect_str) as conn:
        query = """SET geqo_effort to 10;
                    SET max_parallel_workers_per_gather TO 8;
                    SET temp_buffers to "32GB";
                    SET work_mem to "1GB";
                    SET temp_tablespaces to "ssdspace";
                SELECT * FROM alive_bees_2016
               ;"""
    df = pd.read_sql_query(
        query, conn, coerce_float=False)

    df = df.groupby('bee_id')['timestamp'].agg(['max'])
    df.to_csv(detections_cache_path+'Last_day_alive.csv')
    print('Saved last day alive for all bees')

def cache_first_detection_dates():
    """Pulls alive_bees_2016 from the Beesbook database
    and saves a dataframe with the first detection date for each bee to disk as "First_day_alive.csv" file.
    WARNING: these are not the bees' birthdays - they might have hatched before the video recordings started.
    Only meta.get_hatchdate is a reliable way of finding birthdays.
    """

    print('Loading bee alive states')

    with psycopg2.connect(connect_str) as conn:
        query = """SET geqo_effort to 10;
                    SET max_parallel_workers_per_gather TO 8;
                    SET temp_buffers to "32GB";
                    SET work_mem to "1GB";
                    SET temp_tablespaces to "ssdspace";
                SELECT * FROM alive_bees_2016
               ;"""
    df = pd.read_sql_query(
        query, conn, coerce_float=False)

    df = df.groupby('bee_id')['timestamp'].agg(['min'])
    df.to_csv(detections_cache_path+'First_day_alive.csv')
    print('Saved first day alive for all bees')

def last_days_caches(num_hours, num_intervals_per_hour, number_last_days):
    #step 1 loading last alive csv
    last_alive_path = detections_cache_path+'Last_day_alive.csv'
    last_day_alive_df = pd.read_csv(last_alive_path,
                               parse_dates=['max'],
                               usecols=['max', 'bee_id'])

    last_day_alive_df = last_day_alive_df.loc[last_day_alive_df['max'] >= datetime(2016, 7, 20+number_last_days)]
    last_day_alive_df.index = last_day_alive_df['bee_id']
    last_days_presence_df = pd.DataFrame()
    last_days_locations_df = pd.DataFrame()
    last_days_locations_front_df = pd.DataFrame()
    last_days_locations_back_df = pd.DataFrame()

    #step 2 traversing through a loop to extract the relevant intervals from each saved cache (traverse over timestamp or bee_id?)
    for ix in tqdm(last_day_alive_df['bee_id']):
        temp_presence_df = pd.DataFrame()
        temp_locations_df = pd.DataFrame()
        temp_locations_front_df = pd.DataFrame()
        temp_locations_back_df = pd.DataFrame()
        for day in range(number_last_days):
            (presence_name, presence_path) = create_presence_cache_filename(last_day_alive_df.loc[ix][1].date()-datetime.timedelta(days=day+1), num_hours, num_intervals_per_hour)
            (locations_name, locations_path) = create_presence_locations_cache_filename(last_day_alive_df.loc[ix][1].date()-datetime.timedelta(days=day+1), num_hours, num_intervals_per_hour)
            (locations_front_name, locations_front_path) = create_presence_locations_cam_cache_filename(last_day_alive_df.loc[ix][1].date()-datetime.timedelta(days=day+1), num_hours, num_intervals_per_hour, "front/")
            (locations_back_name, locations_back_path) = create_presence_locations_cam_cache_filename(last_day_alive_df.loc[ix][1].date()-datetime.timedelta(days=day+1), num_hours, num_intervals_per_hour, "back/")

            new_presence_df = pd.read_csv(presence_path)
            new_presence_df = new_presence_df.loc[new_presence_df['id'] == ix]
            new_locations_df = pd.read_csv(locations_path, low_memory=False)
            new_locations_df = new_locations_df.loc[new_locations_df['id'] == ix]
            new_locations_front_df = pd.read_csv(locations_front_path, low_memory=False)
            new_locations_front_df = new_locations_front_df.loc[new_locations_front_df['id'] == ix]
            new_locations_back_df = pd.read_csv(locations_back_path, low_memory=False)
            new_locations_back_df = new_locations_back_df.loc[new_locations_back_df['id'] == ix]

            if temp_presence_df.empty:
                temp_presence_df = new_presence_df.iloc[:,1:]
            else:
                temp_presence_df = temp_presence_df.merge(new_presence_df.iloc[:,1:], on='id', how='left')
                temp_presence_df.columns = range(-1, temp_presence_df.shape[1]-1)
                temp_presence_df.columns = temp_presence_df.columns.map(str)
                temp_presence_df.rename(columns={'-1': 'id'}, inplace=True)

            if temp_locations_df.empty:
                temp_locations_df = new_locations_df.iloc[:,1:]
            else:
                temp_locations_df = temp_locations_df.merge(new_locations_df.iloc[:,1:], on='id', how='left')
                temp_locations_df.columns = range(-1, temp_locations_df.shape[1]-1)
                temp_locations_df.columns = temp_locations_df.columns.map(str)
                temp_locations_df.rename(columns={'-1': 'id'}, inplace=True)

            if temp_locations_front_df.empty:
                temp_locations_front_df = new_locations_front_df.iloc[:,1:]
            else:
                temp_locations_front_df = temp_locations_front_df.merge(new_locations_front_df.iloc[:,1:], on='id', how='left')
                temp_locations_front_df.columns = range(-1, temp_locations_front_df.shape[1]-1)
                temp_locations_front_df.columns = temp_locations_front_df.columns.map(str)
                temp_locations_front_df.rename(columns={'-1': 'id'}, inplace=True)

            if temp_locations_back_df.empty:
                temp_locations_back_df = new_locations_back_df.iloc[:,1:]
            else:
                temp_locations_back_df = temp_locations_back_df.merge(new_locations_back_df.iloc[:,1:], on='id', how='left')
                temp_locations_back_df.columns = range(-1, temp_locations_back_df.shape[1]-1)
                temp_locations_back_df.columns = temp_locations_back_df.columns.map(str)
                temp_locations_back_df.rename(columns={'-1': 'id'}, inplace=True)

        if last_days_presence_df.empty:
            last_days_presence_df = temp_presence_df
        else:
            last_days_presence_df = pd.concat([last_days_presence_df, temp_presence_df])

        if last_days_locations_df.empty:
            last_days_locations_df = temp_locations_df
        else:
            last_days_locations_df = pd.concat([last_days_locations_df, temp_locations_df])

        if last_days_locations_front_df.empty:
            last_days_locations_front_df = temp_locations_front_df
        else:
            last_days_locations_front_df = pd.concat([last_days_locations_front_df, temp_locations_front_df])

        if last_days_locations_back_df.empty:
            last_days_locations_back_df = temp_locations_back_df
        else:
            last_days_locations_back_df = pd.concat([last_days_locations_back_df, temp_locations_back_df])

    print (last_days_presence_df)
    print (last_days_locations_df)
    print (last_days_locations_front_df)
    print (last_days_locations_back_df)

    #step 3 saving the caches
    #NOTE: make sure the paths are correct
    last_days_presence_df.to_csv(cache_location_prefix+'Presence/Janek/last_days_presence.csv')
    last_days_locations_df.to_csv(cache_location_prefix+'Presence/Janek/locations/last_days_locations.csv')
    last_days_locations_front_df.to_csv(cache_location_prefix+'Presence/Janek/locations/cam/front/last_days_locations_front.csv')
    last_days_locations_back_df.to_csv(cache_location_prefix+'Presence/Janek/locations/cam/back/last_days_locations_back.csv')
    print('Saved last day caches for all bees')

def calculate_bee_lifespans_from_hatchdates():
    """
    Uses the Last_day_alive.csv cache and the get_hatchdate functions from bb_utils
    to calculate the lifespan (length-of-life) in days for each bee. The lifespan will be NaN if hatchdate is NaT.

    """
    meta = bb_utils.meta.BeeMetaInfo()
    last_alive_path = detections_cache_path+'Last_day_alive.csv'
    last_day_alive_df = pd.read_csv(last_alive_path,
                                    parse_dates=['max'],
                                    usecols=['max', 'bee_id'])

    last_day_alive_df.index = last_day_alive_df['bee_id']
    lifespan = pd.DataFrame()
    for ix in last_day_alive_df['bee_id']:
        birthday = meta.get_hatchdate(bb_utils.ids.BeesbookID.from_dec_9(ix)).date()
        temp_lifespan = pd.DataFrame([[ix, (last_day_alive_df.loc[ix]['max'].date() - birthday).days]], columns=['id','lifespan'])
        lifespan = pd.concat([lifespan, temp_lifespan])
    lifespan.index = last_day_alive_df['bee_id']
    lifespan = lifespan['lifespan']
    lifespan = filter_out_fake_deaths(lifespan)
    return lifespan

#TODO: returns a Series, while other two lifespan functions return a Dateframe
def calculate_bee_lifespans_from_detections():
    last_alive_path = detections_cache_path+'Last_day_alive.csv'
    first_alive_path = detections_cache_path+'First_day_alive.csv'

    last_day_alive_df = pd.read_csv(last_alive_path, parse_dates=['max'], usecols=['max', 'bee_id'])
    first_day_alive_df = pd.read_csv(first_alive_path, parse_dates=['min'], usecols=['min'])

    diff = (last_day_alive_df['max'] - first_day_alive_df['min']).dt.days
    diff.index = last_day_alive_df['bee_id']
    diff = filter_out_fake_deaths(diff)
    return diff

def calculate_bee_lifespans_combined():
    last_alive_path = detections_cache_path+'Last_day_alive.csv'
    first_alive_path = detections_cache_path+'First_day_alive.csv'

    last_day_alive_df = pd.read_csv(last_alive_path,
                                    parse_dates=['max'],
                                    usecols=['max', 'bee_id'])
    first_day_alive_df = pd.read_csv(first_alive_path,
                                     parse_dates=['min'],
                                     usecols=['min', 'bee_id'])

    last_day_alive_df.index = last_day_alive_df['bee_id']
    first_day_alive_df.index = first_day_alive_df['bee_id']

    meta = bb_utils.meta.BeeMetaInfo()

    lifespan_from_detections = calculate_bee_lifespans_from_detections()

    lifespan = pd.DataFrame()
    for ix in last_day_alive_df['bee_id']:
        birthday = meta.get_hatchdate(bb_utils.ids.BeesbookID.from_dec_9(ix)).date()
        if pd.isnull(birthday):
            birthday = first_day_alive_df.loc[ix]['min'].date()
        temp_lifespan = pd.DataFrame([[ix, (last_day_alive_df.loc[ix]['max'].date() - birthday).days]], columns=['id','lifespan'])
        lifespan = pd.concat([lifespan, temp_lifespan])
    lifespan.index = last_day_alive_df['bee_id']
    lifespan = lifespan['lifespan']
    lifespan = filter_out_fake_deaths(lifespan)
    return lifespan

def filter_out_fake_deaths(lifespans):
    """Takes a lifespans dataframe and filters out bee ids
    whose last detections landed on the last day of the experiment,
    because they shouldn't be interpreted as deaths."""
    last_alive_path = detections_cache_path+'Last_day_alive.csv'
    last_day_alive_df = pd.read_csv(last_alive_path,
                                    parse_dates=['max'],
                                    usecols=['max', 'bee_id'])

    last_date = last_day_alive_df['max'].max()
    bees_with_fake_deaths = last_day_alive_df['bee_id'].drop(last_day_alive_df[last_day_alive_df['max'] != last_date].index)
    return lifespans.drop(bees_with_fake_deaths)


from sys import path
from PathPlanning import PathPlanning
from MoveCalculation import MoveCalculation
from MoveSimulation import Simulation
import serial
import time

class Toolbox(PathPlanning, MoveCalculation, Simulation):
    pass

a = Toolbox()
a.setState("A")
# a.simulation(a.pathPlanningStep("D"))
run = a.pathPlanningStep("D", status=False)
counterA = 0
#print(run)

def rencana () :
    for i in range(len(run)):
        if i<1:
            path = str((run[counterA][0]))
            jarak = str((run[counterA][1]))
            derajat = str((run[counterA][2]))
            tujuan = str((run[counterA][3]))

    total = path + ',' + jarak + ',' + derajat + ',' + tujuan + '$'
    return total

# #cam = 'krai' + '#' + ','
if __name__ == '__main__':
    ser = serial.Serial('COM13',115200, timeout=1)
    ser.flush()
    while True:
        if ser.in_waiting > 0:
            line = ser.readline().decode('utf-8').rstrip()
            if counterA > len(run) :
                counterA=0
            if line == "Path Selesai":
                counterA +=1
            ser.write(b"" + rencana().encode())
            print(rencana())
            print(line)


from pathlib import Path

BASE_DIR = Path(__file__).resolve().parent
STATIC_ROOT = BASE_DIR / 'static'

...

# Adicione esta linha para especificar a pasta static como um diret�rio est�tico
STATICFILES_DIRS = [STATIC_ROOT]

...

workers = 4  # ajuste o número de trabalhadores aqui
worker_connections = 1000


import sys, unittest
from functools import reduce
from faker import Faker
sys.path.insert(0, './src')
from classes import MealBox
from resources import *

faker = Faker()

class MealBoxTest(unittest.TestCase):

   def test_mealBox_passing_argument(self):
      argument = faker.name()

      with self.assertRaises(TypeError):
         meal_box = MealBox(argument)

   def test_mealBox_instancing(self):
      meal_box = MealBox()
      self.assertTrue(isinstance(meal_box, MealBox))

   def test_mealValidate_with_random_value(self):
      argument_1 = faker.name()
      argument_2 = faker.random_int()

      with self.assertRaises(Exception):
         MealBox._meal_validate(argument_1)

      with self.assertRaises(Exception):
         MealBox._meal_validate(argument_2)

      with self.assertRaises(Exception):
         MealBox._meal_validate(None)

   def test_mealValidate_with_spy_without_name(self):
      spy = AbstractMealWithOutNameSpy()

      with self.assertRaises(Exception):
         MealBox._meal_validate(spy)

   def test_mealValidate_with_spy_without_price(self):
      spy = AbstractMealWithOutPriceSpy()

      with self.assertRaises(Exception):
         MealBox._meal_validate(spy)

   def test_mealValidate_with_spy_without_get_price(self):
      spy = AbstractMealWithOutGetPriceSpy()

      with self.assertRaises(Exception):
         MealBox._meal_validate(spy)

   def test_add_with_empty_list(self):
      meal_box = MealBox()
      meal_box.add(list())

      self.assertFalse(bool(meal_box._children))

   def test_add_with_single_spy(self):
      spy = AbstractMealSpy()
      meal_box = MealBox()
      meal_box.add(spy)

      SPY_INDEX = 0

      self.assertTrue(bool(meal_box._children)) # is not empty
      self.assertEqual(meal_box._children[SPY_INDEX], spy)

   def test_add_with_multiple_spies(self):
      spies_number = faker.random_int(min=2,max=30)
      spies = [AbstractMealSpy() for i in range(spies_number)]
      meal_box = MealBox()
      meal_box.add(spies)

      self.assertTrue(bool(meal_box._children)) # is not empty
      self.assertEqual(len(meal_box._children), len(spies))

      for value, spy in zip(meal_box._children, spies):
         self.assertEqual(value, spy)

   def test_get_price_with_no_values_add(self):
      meal_box = MealBox()
      EMPTY_PRICE = 0

      self.assertEqual(meal_box.get_price(), EMPTY_PRICE)

   def test_get_price_with_single_spy(self):
      spy = AbstractMealSpy()
      meal_box = MealBox()
      meal_box.add(spy)

      self.assertEqual(meal_box.get_price(), int(spy.price))

   def test_get_price_with_multiple_spies(self):
      spies_number = faker.random_int(min=2,max=30)
      spies = [AbstractMealSpy() for _ in range(spies_number)]
      total_price = reduce(lambda sum, spy: sum + spy.get_price(), spies, 0)
      meal_box = MealBox()
      meal_box.add(spies)

      self.assertEqual(meal_box.get_price(), total_price)

if __name__ == '__main__':
   unittest.main()


# -*- coding: utf-8 -*-

from osv import osv, fields
from datetime import datetime

from ..giscedata_polissa import PERMISSION_SELECTION, ERROR_CODES
AVAILABLE_STATES = [
    ('init','Init'),
    ('admin_specified', 'Nova administradora OV especificada'),
    ('done', 'Done')
]

class WizardModifyOVAdmin(osv.osv_memory):

    _name = 'wizard.modify.ov.admin'

    def onchange_administradora(self, cursor, uid, ids, administradora_id):
            res = {'value': {"state": "init"}}
            if administradora_id:
                res = {'value': {"state": "admin_specified"}}
            return res

    def default_get(self, cr, uid, fields, context={}):
        active_id = context.get('active_id', False)

        res = {}
        if not active_id:
            return res

        if isinstance(active_id, (list, tuple)):
            active_id = active_id[0]

        pol_obj = self.pool.get("giscedata.polissa")
        titular = pol_obj.read(cr, uid, active_id, ['titular'])['titular'][0]

        res['claimant'] = titular
        res['state'] = 'init'

        return res

    def annotate_legal_representative(self, cursor, uid, titular_id, repersentative_id):
        res_partner_obj = self.pool.get('res.partner')
        admin_comment = res_partner_obj.read(cursor, uid, titular_id,
            ['comment']).get("comment", "")
        representative_vals = res_partner_obj.read(cursor, uid, repersentative_id,
            ['name', 'vat'])
        data = datetime.now().strftime('%Y-%m-%d %H:%M:%S')
        message = u"Durant el procés d'assignació d'Administradora OV, aquest " +\
            u"partner va declarar en {} que el partner ".format(data) +\
            u"{} amb VAT {} és el representant legal d'aquest.".format(
                representative_vals['name'], representative_vals['vat']
            )
        if admin_comment:
            message = admin_comment + "\n\n" + message
        res_partner_obj.write(cursor, uid, titular_id,
            {"comment": message})

    def create_admin_notification_modification(self, cursor, uid, partner_id,
            ov_admin_modification, notification_receptor, semantic_id):
        if partner_id is None:
            return

        mdata_obj = self.pool.get('ir.model.data')
        template_id = mdata_obj.get_object_reference(
            cursor, uid, 'som_polissa_administradora', semantic_id
        )[1]

        admin_noti_obj = self.pool.get('som.admin.notification')
        if partner_id not in notification_receptor.keys():
            notification_receptor[partner_id] = admin_noti_obj.create(cursor, uid, {
                "receptor": partner_id, "pending_notification": True,
                "template_id": template_id
            })

        admin_noti_obj.write(cursor, uid, notification_receptor[partner_id],
            {'modification': [(4, ov_admin_modification)]})

    def action_assignar_administradora(self, cursor, uid, ids, context=None):
        active_ids = context.get('active_ids', [])

        if not active_ids:
            return {}

        pol_obj = self.pool.get('giscedata.polissa')
        partner_obj = self.pool.get('res.partner')
        admin_noti_obj = self.pool.get("som.admin.notification")
        admin_mod_obj = self.pool.get("som.admin.modification")

        wiz_vals = self.read(cursor, uid, ids)[0]
        new_administradora_id = wiz_vals.get("new_administradora", False)
        administradora_permissions = wiz_vals.get("administradora_permissions", False)
        is_legal_representative = wiz_vals.get("is_legal_representative", False)
        not_notify_changes = wiz_vals.get("not_notify_changes", False)

        # validate fields
        if not (new_administradora_id and administradora_permissions):
            raise osv.except_osv('Error!', u"Falten camps per emplenar")

        new_administradora_name = partner_obj.read(
            cursor, uid, new_administradora_id, ['name'])['name']

        notification_receptor = {}
        titulars_anotats = []
        polisses = pol_obj.browse(cursor, uid, active_ids)
        info = ""
        for pol in polisses:
            # keep old administrator if there is
            old_administradora_id = None
            if pol.administradora:
                old_administradora_id = pol.administradora.id

            titular_id = pol.titular.id

            # perform assignment
            result = pol.add_contract_administrator(new_administradora_id,
                    administradora_permissions, is_legal_representative=is_legal_representative)

            error = result.get('error', None) is not None

            ov_admin_modification = None
            if error:
                error_msg = u"- S'ha produït un error al assignar l'administradora " + \
                    u"per la pòlissa {}: \"{}\"\n"
                error_msg = error_msg.format(pol.name, result['error_msg'])
                info += error_msg

                skip_error = ERROR_CODES.get(result['error'], {}).get('skip_error', False)
                if not skip_error:
                    vals = {
                        "old_administradora": old_administradora_id,
                        "new_administradora": new_administradora_id,
                        "permissions": administradora_permissions,
                        "is_legal_representative": is_legal_representative,
                        "error": True,
                        "info": error_msg
                    }
                    ov_admin_modification = pol_obj.register_ov_admin_modification(
                        cursor, uid, pol.id, vals)
                else:
                    continue
            else:
                ov_admin_modification = result['modification']
                # update info if there's no error
                msg = ''
                if old_administradora_id is not None:
                    old_administradora_name = partner_obj.read(
                        cursor, uid, old_administradora_id, ['name'])['name']
                    msg = u"- S'ha sustituit el partner '{}' (ID {}) pel partner " + \
                        u"'{}' (ID {}) com a administradora OV a la pòlissa {}\n"
                    msg = msg.format(
                        old_administradora_name, old_administradora_id,
                        new_administradora_name, new_administradora_id, pol.name)
                else:
                    msg = u"- S'ha assignat el partner '{}' (ID {}) " + \
                        u"com a administradora OV a la pòlissa {}\n"
                    msg = msg.format(
                        new_administradora_name, new_administradora_id, pol.name)

                admin_mod_obj.write(cursor, uid, ov_admin_modification, {"info": msg})

                info += msg

            # create notifications and associate with modification
            self.create_admin_notification_modification(cursor, uid, titular_id,
                ov_admin_modification, notification_receptor, 'email_assignacio_a_titular')
            self.create_admin_notification_modification(cursor, uid, old_administradora_id,
                ov_admin_modification, notification_receptor, 'email_desassignacio_a_administradora')
            self.create_admin_notification_modification(cursor, uid, new_administradora_id,
                ov_admin_modification, notification_receptor, 'email_assignacio_a_administradora')

            if is_legal_representative and titular_id not in titulars_anotats:
                self.annotate_legal_representative(cursor, uid, titular_id, new_administradora_id)
                titulars_anotats.append(titular_id)

        self.write(cursor, uid, ids, {
            "state": "done",
            "info": info
        })

        cursor.commit()

        if not not_notify_changes:
            # notify assignation
            for noti_id in notification_receptor.values():
                admin_noti_obj.send_email(cursor, uid, noti_id)

        return

    def action_desassignar_administradora(self, cursor, uid, ids, context=None):
        active_ids = context.get('active_ids', [])

        if not active_ids:
            return {}

        pol_obj = self.pool.get('giscedata.polissa')
        partner_obj = self.pool.get('res.partner')
        admin_noti_obj = self.pool.get("som.admin.notification")
        admin_mod_obj = self.pool.get("som.admin.modification")

        wiz_vals = self.read(cursor, uid, ids)[0]
        not_notify_changes = wiz_vals.get("not_notify_changes", False)

        notification_receptor = {}
        polisses = pol_obj.browse(cursor, uid, active_ids)
        info = ""
        for pol in polisses:
            # keep old administrator if there is
            old_administradora_id = None
            if pol.administradora:
                old_administradora_id = pol.administradora.id

            titular_id = pol.titular.id

            # perform unassignment
            result = pol.remove_contract_administrator()

            error = result.get('error', None) is not None

            ov_admin_modification = None
            if error:
                error_msg = u"- S'ha produït un error al assignar l'administradora " + \
                    u"per la pòlissa {}: \"{}\"\n"
                error_msg = error_msg.format(pol.name, result['error_msg'])
                info += error_msg

                skip_error = ERROR_CODES.get(result['error'], {}).get('skip_error', False)
                if not skip_error:
                    vals = {
                        "old_administradora": old_administradora_id,
                        "new_administradora": None,
                        "permissions": None,
                        "is_legal_representative": False,
                        "error": True,
                        "info": error_msg
                    }
                    ov_admin_modification = pol_obj.register_ov_admin_modification(
                        cursor, uid, pol.id, vals)
                else:
                    continue
            else:
                ov_admin_modification = result['modification']
                # update info if there's no error
                old_administradora_name = partner_obj.read(
                    cursor, uid, old_administradora_id, ['name'])['name']
                msg = u"- S'ha desassignat el partner '{}' (ID {}) de la pòlissa {}\n".format(
                    old_administradora_name, old_administradora_id, pol.name)

                admin_mod_obj.write(cursor, uid, ov_admin_modification, {"info": msg})

                info += msg

            # create notifications and associate with modification
            self.create_admin_notification_modification(cursor, uid, titular_id,
                ov_admin_modification, notification_receptor, 'email_desassignacio_a_titular')
            self.create_admin_notification_modification(cursor, uid, old_administradora_id,
                ov_admin_modification, notification_receptor, 'email_desassignacio_a_administradora')

        self.write(cursor, uid, ids, {
            "state": "done",
            "info": info
        })

        cursor.commit()

        if not not_notify_changes:
            # notify assignation
            for noti_id in notification_receptor.values():
                admin_noti_obj.send_email(cursor, uid, noti_id)

        return

    _columns = {
        'new_administradora': fields.many2one(
            'res.partner', 'Nova Administradora OV'
        ),
        'claimant': fields.many2one(
            'res.partner', 'Sol·licitant'
        ),
        'is_legal_representative': fields.boolean(
            'Representant legal',
            help=u"Si s'activa, es deixarà una nota a la fitxa del titular amb el nom i VAT " +\
                u"de l'Administradora OV, especificant que aquesta última és la representant " +\
                u"legal del titular."
        ),
        'administradora_permissions': fields.selection(
            PERMISSION_SELECTION,
            string='Permisos Administradora OV',
            help=u"Permisos de l'Administradora OV sobre la pòlissa"
        ),
        'state': fields.selection(
            selection=AVAILABLE_STATES,
            string='Estat'
        ),
        'not_notify_changes': fields.boolean(
            'No notificar canvis',
            help=u"Si s'activa, els canvis d'Administradora no es notificaràn a ningú"
        ),
        'info': fields.text('Info'),
    }

WizardModifyOVAdmin()


# contains several functions for comparing the gaze data as heatmaps or fixation location map or list of locations with a saliency heatmap

from sklearn.metrics import roc_auc_score
import tensorflow as tf
import numpy as np
from scipy import ndimage as ndi
from statsmodels.stats.weightstats import ztest

def heatmap_comparison_using_AUC(gaze, saliency):
    """ binary classifier: Area Under ROC Curve

    Parameters:
        gaze (heatmap or location map): heatmap -> AUC, location map -> AUC-Judd
        saliency (heatmap): classifier
    """

    gaze_flat = to_binary_flat(gaze)
    saliency_flat = to_binary_flat(saliency)

    auc_score = roc_auc_score(gaze_flat, saliency_flat)

    return auc_score

def heatmap_comparison_percentage_saliency_also_true(gaze,saliency):
    """ calculates the percentage of points being one from the saliency map, that are one on the gaze heatmap (True Positive) """

    gaze_flat = to_binary_flat(gaze)
    sal_flat = to_binary_flat(saliency)

    sal_reduced = np.delete(sal_flat, np.where(gaze_flat == 0))

    percentage_true = np.sum(sal_reduced) / sal_reduced.shape[0]
    return percentage_true

def heatmap_comparison_percentage_same(gaze,saliency):

    gaze_flat = to_binary_flat(gaze)
    sal_flat = to_binary_flat(saliency)

    total = gaze_flat.shape[0] # 33600
    same = np.where(gaze_flat == sal_flat,1,0) # condition, true, false
    return np.sum(same) / total

def to_binary_flat(map):
    """ creates a binary flat map from a heatmap """
    # make map between 0 and 1
    map_max = tf.reduce_max(map)
    map_normal = map / map_max
    # make map only 0 or 1
    map_rounded = round_with_threshold(map_normal)
    # flatten
    map_flat = tf.reshape(map_rounded,[-1]).numpy()
    return map_flat

def round_with_threshold(array,threshold=0.2, min=0, max=1):
    return np.where(array > threshold, max, min)

def saliency_information_gain(fixation_map, saliency,epsilon=0.1):
    """ compare by using Information Gain

    Arguments:
        fixation_map (2D array or Tensor): binary map of fixation points (not a heatmap)
        saliency: saliency heatmap
        epsilon (float): for regularization
    """
    center_prior = create_center_prior_baseline(fixation_map)

    log_saliency = np.log2(saliency + epsilon)
    log_prior = np.log2(center_prior + epsilon)
    n = tf.reshape(fixation_map,[-1]).shape[0]

    # calculation
    information_gain = ( 1/n ) * tf.reduce_sum(tf.multiply(fixation_map,log_saliency - log_prior))
    return information_gain.numpy()

def create_center_prior_baseline(map):
    """ creates a center prior baseline in the shape of map """
    base = np.zeros(shape=map.shape)
    center_x = base.shape[0] // 2
    center_y = base.shape[1] //2
    base[center_x,center_y] = 1
    base = ndi.gaussian_filter(base, sigma = 7) # compare my_reader_class.create_gaze_heatmap
    return base

def compare_by_mean(gaze_list : list ,saliency_map):
    """ calculates the distance of the mean value for the gaze list and a saliency map. the map made into a binary map first and only 1 values are used for the mean.
    it also outputs the distance of the gaze to the middle of the image, to see if that would have been a better classifier (only good prediction if dist to saliency smaller than to middle)
    """

    s_map_max = tf.reduce_max(saliency_map)
    s_map_normal = saliency_map / s_map_max
    s_map_rounded = round_with_threshold(s_map_normal)

    gaze_mean = np.mean(gaze_list)
    saliency_list = []
    for x in range(saliency_map.shape[0]):
        for y in range(saliency_map.shape[1]):
            if s_map_rounded[x][y] == 1:
                saliency_list.append((x,y))
    saliency_mean = np.mean(saliency_list)

    # check if the middle would have been a better guess
    numerator = np.mean(saliency_map.shape)
    dist_middle = np.linalg.norm(gaze_mean - np.array([int(saliency_map.shape[0]/2),int(saliency_map.shape[1]/2)])) / numerator
    dist_sal = np.linalg.norm(gaze_mean - saliency_mean) / numerator

    return dist_sal, dist_middle

def calc_correlation(gaze_map,saliency_map):
    """ compares how similar the two input maps are by using correlation """
    gaze_flat = gaze_map.flatten()
    sal_flat = saliency_map.flatten()
    correlation_matrix = np.corrcoef(gaze_flat,sal_flat)
    return correlation_matrix[0,1]

def heatmap_correlation(gaze_map,saliency_map):
    """ calculates a correlation heatmap for the given input
    both should contains floats in the range of 0-1 """

    cor_heatmap = tf.divide(tf.multiply(saliency_map,gaze_map),  tf.sqrt(tf.reduce_sum( tf.add(tf.square(saliency_map), tf.square(gaze_map)) ) ) )
    return cor_heatmap

def z_test(x,y,alternative='both-sided'):
    """ source: https://stackoverflow.com/questions/61379874/how-to-perform-two-sample-one-tailed-t-test-in-python """

    _, double_p = ztest(x, y, value=0) # value = differenz between the two under H0
    pval = -1
    if alternative == 'both-sided':
        pval = double_p
    elif alternative == 'greater':
        if np.mean(x) > np.mean(y):
            pval = double_p/2.
        else:
            pval = 1.0 - double_p/2.
    elif alternative == 'less':
        if np.mean(x) < np.mean(y):
            pval = double_p/2.
        else:
            pval = 1.0 - double_p/2.
    return pval