#Packages
import pandas as pd
import numpy as np
import pyreadr as pyr
import sklearn as sk
import scipy as sc
from sklearn.linear_model import LinearRegression
from sklearn.model_selection import train_test_split
from statsmodels.formula.api import ols
import statsmodels.api as sm
from statsmodels.stats.outliers_influence import OLSInfluence
from sklearn.metrics import mean_squared_error
from matplotlib import pyplot as plt
import seaborn as sns
from sklearn.model_selection import LeaveOneOut
from sklearn.model_selection import cross_val_score
from bootstrap_stat import bootstrap_stat as bp
from scipy.stats import bootstrap
from sklearn.impute import KNNImputer

#Setting work space and master data frame:
bd=pyr.read_r("bdPS1.Rdata")
print(bd.keys()) #Load database.
df0=bd["df"]
df=df0.drop(df0[df0.age<18].index) #Restrict the data to legal age individuals (age>=18).
df=df.drop(df[df.ocu==0].index) #Restrict the data to employed individuals (ocu==1).
dfsi=(df[["age", "maxEducLevel", "sizeFirm", "oficio", "hoursWorkUsual", "formal", "ingtot", "mes", "estrato1", "sex",
"regSalud", "p6426", "p7090","p7510s7a1", "impa", "isa", "impaes", "isaes", "relab"]]) #Relevant variables selection.

#Reassining variables codification:
dfsi["impa"].replace(0, 1, inplace=True)
dfsi["impa"].replace(np.nan, 0, inplace=True)
dfsi["impaes"].replace(0, 1, inplace=True)
dfsi["impaes"].replace(np.nan, 0, inplace=True)
dfsi["isa"].replace(0, 1, inplace=True)
dfsi["isa"].replace(np.nan, 0, inplace=True)
dfsi["isaes"].replace(0, 1, inplace=True)
dfsi["isaes"].replace(np.nan, 0, inplace=True)

#Create compiled variables:
dfsi["impacomp"]=dfsi["impa"]+dfsi["impaes"]
dfsi["isacomp"]=dfsi["isa"]+dfsi["isaes"]

#Recover missing values:
dfsi['impacomp']=dfsi['impacomp'].replace(0,np.nan)
dfsi['isacomp']=dfsi['isacomp'].replace(0,np.nan)
dfsi['impacomp']=dfsi['impacomp'].replace(1, 0)
dfsi['isacomp']=dfsi['isacomp'].replace(1, 0)
dfsi['impacomp']=dfsi['impacomp'].replace(2, 0)
dfsi['isacomp']=dfsi['isacomp'].replace(2, 0)

#Missing values count/share.
dfsi.isnull().sum()
dfsi.isnull().sum()/len(dfsi)

#Imputing missing values:
imputer=KNNImputer(n_neighbors=19)
imputer.fit(dfsi)
dfi=pd.DataFrame(imputer.transform(dfsi))
dict=({0: "age", 1: "maxEducLevel", 2: "sizeFirm", 3: "oficio", 4: "hoursWorkUsual", 5: "formal", 6: "ingtot", 7: "mes", 8: "estrato1",
9: "sex", 10: "regSalud", 11: "p6426", 12: "p7090", 13: "p7510s7a1", 14: "impa", 15: "isa", 16: "impaes", 17: "isaes", 18: "relab",
19: "impacomp", 20: "isacomp"})
dfi.rename(columns=dict, inplace=True)

#Convert categorical variables to dummy variables:
oficio_d=pd.get_dummies(dfi["oficio"], prefix="oficio")
sizeFirm_d=pd.get_dummies(dfi["sizeFirm"], prefix="size")
estrato1_d=pd.get_dummies(dfi["estrato1"], prefix="estrato")
regSalud_d=pd.get_dummies(dfi["regSalud"], prefix="regs")
maxeduc_d=pd.get_dummies(dfi["maxEducLevel"], prefix="educ")
relab_d=pd.get_dummies(dfi["relab"], prefix="relab")
mes_d=pd.get_dummies(dfi["mes"], prefix="mes")

#Merge dummy's variables data frame with master data frame:
dfi=pd.merge(dfi, oficio_d, left_index=True, right_index=True)
dfi=pd.merge(dfi, sizeFirm_d, left_index=True, right_index=True)
dfi=pd.merge(dfi, estrato1_d, left_index=True, right_index=True)
dfi=pd.merge(dfi, regSalud_d, left_index=True, right_index=True)
dfi=pd.merge(dfi, maxeduc_d, left_index=True, right_index=True)
dfi=pd.merge(dfi, relab_d, left_index=True, right_index=True)
dfi=pd.merge(dfi, mes_d, left_index=True, right_index=True)
dfi["p7090"]=np.where(dfi["p7090"]==2, 0, 1)

#Transform categorial atribute variables to numerical variable:
dfi["hoursWorkUsual"]=pd.to_numeric(dfi["hoursWorkUsual"])
dfi["formal"]=pd.to_numeric(dfi["formal"])
dfi["p6426"]=pd.to_numeric(dfi["p6426"])

#Descriptive statistics data frame:
ds=(dfi[["age", "hoursWorkUsual", "p6426", "sex", "formal", "p7090", "estrato_1.0", "estrato_2.0", "estrato_3.0",
"estrato_4.0", "estrato_5.0", "estrato_6.0", "regs_1.0", "regs_2.0", "regs_3.0", "educ_1.0", "educ_3.0", "educ_4.0",
"educ_5.0", "educ_6.0", "educ_7.0"]].describe(include="all"))
ds=ds.T
ds=ds[["count", "mean", "std", "min", "50%", "max"]]
ds=ds.round(2)
print(ds.to_latex())

#Create dependent variable:
dfi["earnings"]=dfi["impacomp"]+dfi["isacomp"]
dfi["earnings"].describe()

#Winsorize data:
sc.stats.mstats.winsorize(dfi["earnings"], limits=[0,0.05], inplace=True)
sc.stats.mstats.winsorize(dfi["age"], limits=[0,0.05], inplace=True)
dfi["earnings"].describe()

#Create cuadratic age variable:
dfi["age2"]=dfi["age"]**2

#OLS regresion model:
y=dfi["earnings"]
x=dfi[["age", "age2"]]
x=sm.add_constant(x)
model1=sm.OLS(y,x)
results=model1.fit()
yhatp1=results.predict()
b1=results.params[1]
b2=results.params[2]
elasticidad=-b1/2*b2
print(results.summary())
print(np.sqrt(results.mse_resid))

#Bootstrap
seboot=[0]*3
for j in range(0,3):
    def estimador(dfi):
        y=dfi["earnings"]
        x=dfi[["age", "age2"]]
        x=sm.add_constant(x)
        model1=sm.OLS(y,x)
        results= model1.fit()
        return results.params[j]

    dist = bp.EmpiricalDistribution(dfi)
    sea=bp.standard_error(dist, estimador, B=1000)
    seboot[j]=sea

b0=results.params[0]
b1=results.params[1]
b2=results.params[2]

b0low=b0-seboot[0]*1.96
b1low=b1-seboot[1]*1.96
b2low=b2-seboot[2]*1.96

b0upper=b0+seboot[0]*1.96
b1upper=b1+seboot[1]*1.96
b2upper=b2+seboot[2]*1.96

yhatlow=b0low+b1low*dfi['age']+b2low*dfi['age2']
yhatupper=b0upper+b1upper*dfi['age']+b2upper*dfi['age2']

#Figura 1
x_f1=pd.DataFrame(dfi['age'])
x_f1["Edad"]=x_f1
y_f1=pd.DataFrame(yhatp1)
y_f1["COP"]=y_f1
z_f1=pd.DataFrame(yhatlow)
z_f1["3"]=z_f1
w_f1=pd.DataFrame(yhatupper)
w_f1["4"]=w_f1

plot_data1=pd.concat([x_f1,y_f1,z_f1,w_f1], axis=1, join="inner")
plot_data1

fig, ax = plt.subplots()
ax= sns.lineplot(x='Edad', y='COP', data=plot_data1, label='Ingreso laboral', linewidth=2, color="black")
ax2 = sns.lineplot(x='Edad', y='4', data=plot_data1, label='IC banda superior', linewidth=1, color="gray")
ax1 = sns.lineplot(x='Edad', y='3', data=plot_data1, label='IC banda inferior', linewidth=1, color="gray")

#Create female variable:
dfi["female"]=np.where(dfi["sex"]==1, 0, 1)
dfi=dfi.drop(dfi[dfi.earnings==0].index)
dfi["log_earnings"]=np.log(dfi["earnings"])
dfi["log_earnings"].describe()

#OLS regresion model:
y=dfi["log_earnings"]
x=dfi["female"]
x=sm.add_constant(x)
model2=sm.OLS(y,x)
results= model2.fit()
print(results.summary())
print(np.sqrt(results.mse_resid))

#rMSE to COP units:
y_hat=results.predict()
from math import e
y_hat2=e**(y_hat)
E = dfi["earnings"]-y_hat2
E2 = E**2
MSE=sum(E2)/(len(E2)-2)
print(np.sqrt(MSE))

#Create interactions variable:
dfi["female_age"]=dfi["female"]*dfi["age"]
dfi["female_age2"]=dfi["female"]*dfi["age2"]

#OLS regresion model:
y=dfi["log_earnings"]
x=dfi[["age", "female", "female_age", "female_age2", "age2"]]
x=sm.add_constant(x)
model3=sm.OLS(y,x)
results= model3.fit()
yhatp3=results.predict()
print(results.summary())
results.params

#rMSE to COP units:
y_hat=results.predict()
from math import e
y_hat2=e**(y_hat)
E = dfi["earnings"]-y_hat2
E2 = E**2
MSE=sum(E2)/(len(E2)-6)
print(np.sqrt(MSE))

#Bootstrap
seboot=[0]*6
for j in range(0,6):
    def estimador3(dfi):
        y=dfi["log_earnings"]
        x=dfi[["age", "female", "female_age", "female_age2", "age2"]]
        x=sm.add_constant(x)
        model1=sm.OLS(y,x)
        results= model1.fit()
        return results.params[j]

    dist = bp.EmpiricalDistribution(dfi)
    sea=bp.standard_error(dist, estimador3, B=1000)
    seboot[j]=sea

seboot

for k in range(0,6):
    locals()['b'+str(k)]=results.params[k]
    locals()['b'+str(k)+'low']=locals()['b'+str(k)]-seboot[k]*1.96
    locals()['b'+str(k)+'upper']=locals()['b'+str(k)]+seboot[k]*1.96

yhatlow=b0low+b1low*dfi['age']+b5low*dfi['age2']+b2low*dfi['female']+b3low*dfi['female_age']+b4low*dfi['female_age2']
yhatupper=b0upper+b1upper*dfi['age']+b5upper*dfi['age2']+b2upper*dfi['female']+b3upper*dfi['female_age']+b4upper*dfi['female_age2']

yhatupperh= np.ma.masked_where(dfi['female']==0, yhatupper)
yhatlowh = np.ma.masked_where(dfi['female']==0, yhatlow)

yhatupperm= np.ma.masked_where(dfi['female']==1, yhatupper)
yhatlowm = np.ma.masked_where(dfi['female']==1,yhatlow )

yhatp3m=np.ma.masked_where(dfi['female']==1, yhatp3)
yhatp3h=np.ma.masked_where(dfi['female']==0, yhatp3)

agem=np.ma.masked_where(dfi['female']==1, dfi['age'])
ageh=np.ma.masked_where(dfi['female']==0, dfi['age'])

#Figura 2m
x_f2m=pd.DataFrame(agem)
x_f2m["Edad"]=x_f2m
y_f2m=pd.DataFrame(yhatp3m)
y_f2m["COP"]=y_f2m
z_f2m=pd.DataFrame(yhatlowm)
z_f2m["3"]=z_f2m
w_f2m=pd.DataFrame(yhatupperm)
w_f2m["4"]=w_f2m

plot_data2m=pd.concat([x_f2m,y_f2m,z_f2m,w_f2m], axis=1, join="inner")
plot_data2m

fig, ax = plt.subplots()
ax= sns.lineplot(x='Edad', y='COP', data=plot_data2m, label='Ingreso laboral mujeres', linewidth=2, color="red")
ax2 = sns.lineplot(x='Edad', y='4', data=plot_data2m, label='IC banda superior', linewidth=1, color="orange")
ax1 = sns.lineplot(x='Edad', y='3', data=plot_data2m, label='IC banda inferior', linewidth=1, color="orange")

#Figura 2h
x_f2h=pd.DataFrame(ageh)
x_f2h["Edad"]=x_f2h
y_f2h=pd.DataFrame(yhatp3h)
y_f2h["COP"]=y_f2h
z_f2h=pd.DataFrame(yhatlowh)
z_f2h["3"]=z_f2h
w_f2h=pd.DataFrame(yhatupperh)
w_f2h["4"]=w_f2h

plot_data2h=pd.concat([x_f2h,y_f2h,z_f2h,w_f2h], axis=1, join="inner")
plot_data2h

fig, ax = plt.subplots()
ax= sns.lineplot(x='Edad', y='COP', data=plot_data2h, label='Ingreso laboral hombres', linewidth=2, color="blue")
ax2 = sns.lineplot(x='Edad', y='4', data=plot_data2h, label='IC banda superior', linewidth=1, color="green")
ax1 = sns.lineplot(x='Edad', y='3', data=plot_data2h, label='IC banda inferior', linewidth=1, color="green")

oficio=["oficio_1.0"]
for i in range(2,99):
    oficio.append("oficio_"+str(i)+".0")
    list1=['oficio_10.0', 'oficio_22.0', 'oficio_23.0', 'oficio_24.0', 'oficio_25.0', 'oficio_26.0', 'oficio_27.0', 'oficio_28.0', 'oficio_29.0', 'oficio_46.0', 'oficio_47.0', 'oficio_48.0', 'oficio_64.0', 'oficio_65.0', 'oficio_66.0', 'oficio_67.0', 'oficio_68.0', 'oficio_69.0', 'oficio_71.0']
    oficio2=[i for i in oficio if i not in list1]

#Long regression
#Define exogenous and endogenous variables:
y=dfi["log_earnings"]
x0=dfi[["female", "age", "age2", "female_age", "female_age2"]]
x1=(dfi[["sizeFirm", "hoursWorkUsual", "formal", "p6426", "educ_3.0", "educ_4.0",
"educ_5.0", "educ_6.0", "educ_7.0", "relab_2.0", "relab_3.0" , "relab_4.0", "relab_5.0", "relab_6.0",
"relab_7.0", "relab_8.0" , "relab_9.0", "mes_1.0", "mes_2.0", "mes_3.0", "mes_4.0", "mes_5.0", "mes_6.0",
"mes_7.0", "mes_9.0", "mes_10.0", "mes_11.0", "mes_12.0"]])
x2=dfi[oficio2]
x=pd.concat([x0,x1,x2], axis=1, join="inner")
x=sm.add_constant(x)

#OLS regresion model:
model2=sm.OLS(y,x)
results= model2.fit()
print(results.summary())
print(np.sqrt(results.mse_resid))

#rMSE to COP units:
y_hat=results.predict()
from math import e
y_hat2=e**(y_hat)
E = dfi["earnings"]-y_hat2
E2 = E**2
MSE=sum(E2)/(len(E2)-131)
print(np.sqrt(MSE))

#FWL theorem
#Define exog and endog variables
y=dfi["log_earnings"]
x0=dfi[["female", "age", "age2", "female_age", "female_age2"]]
x_f=pd.concat([x1,x2], axis=1, join="inner")
x_f=sm.add_constant(x_f)
x0=sm.add_constant(x0)

#Residuals of log_earnings regressed on controls
model3=sm.OLS(y,x_f)
results_f= model3.fit()
resid_a=results_f.resid

#Residuals of female regressed on controls
model4=sm.OLS(x0, x_f)
results_f2= model4.fit()
resid_b=results_f2.resid

#Delete constant from residuals b
resid_b=resid_b.drop(["const"], axis=1)

#Results of resid_a  regressed on resid_b
model_FWL=sm.OLS(resid_a,resid_b)
results_FWL=model_FWL.fit()
print(results_FWL.summary())
print(np.sqrt(results.mse_resid))

#rMSE to COP units:
y_hat=results.predict()
from math import e
y_hat2=e**(y_hat)
E = dfi["earnings"]-y_hat2
E2 = E**2
MSE=sum(E2)/(len(E2)-5)
print(np.sqrt(MSE))

#Bootstrap FWL
se_boot = []
for koef in range(0,5):

    def estm(dfi):

        #FWL theorem
        #Define exog and endog variables
        oficio=["oficio_1.0"]
        for i in range(2,99):
            oficio.append("oficio_"+str(i)+".0")
            list1=['oficio_10.0', 'oficio_22.0', 'oficio_23.0', 'oficio_24.0', 'oficio_25.0', 'oficio_26.0', 'oficio_27.0', 'oficio_28.0', 'oficio_29.0', 'oficio_46.0', 'oficio_47.0', 'oficio_48.0', 'oficio_64.0', 'oficio_65.0', 'oficio_66.0', 'oficio_67.0', 'oficio_68.0', 'oficio_69.0', 'oficio_71.0']
            oficio2=[i for i in oficio if i not in list1]

        y=dfi["log_earnings"]
        x0=dfi[['female', "age", "age2", "female_age", "female_age2"]]
        x1=dfi[["sizeFirm", "hoursWorkUsual","formal", "p6426","educ_3.0", "educ_4.0",
            "educ_5.0", "educ_6.0", "educ_7.0", "relab_2.0", "relab_3.0" , "relab_4.0","relab_5.0" , "relab_6.0" , "relab_7.0" , "relab_8.0" , "relab_9.0",
            "mes_1.0", "mes_2.0", "mes_3.0", "mes_4.0", "mes_5.0", "mes_6.0", "mes_7.0","mes_9.0", "mes_10.0", "mes_11.0", "mes_12.0"]]
        x2=dfi[oficio2]
        x_f = pd.concat([x1,x2], axis=1, join="inner")
        x_f=sm.add_constant(x_f)
        x0=sm.add_constant(x0)

        #Residuals of log_earnings regressed on controls
        model3=sm.OLS(y,x_f)
        results_f= model3.fit()
        resid_a=results_f.resid

        #residuals of female regressed on controls
        model4=sm.OLS(x0, x_f)
        results_f2= model4.fit()
        resid_b=results_f2.resid

        #Delete constant from residuals b
        resid_b=resid_b.drop(['const'], axis=1)

        #Results of resid_a  regressed on resid_b
        model_FWL=sm.OLS(resid_a,resid_b)
        results_FWL=model_FWL.fit()

        return results_FWL.params[koef]

    dist = bp.EmpiricalDistribution(dfi)
    se_koef=bp.standard_error(dist,estm, B=5)
    se_boot = se_boot + [str(se_koef)]
    print(se_koef)

#Create new variables
dfi['age_hours']=dfi['age']*dfi['hoursWorkUsual']
dfi['hours2']=dfi['hoursWorkUsual']**2
dfi['age_hours2']=dfi['age']*dfi['hours2']

for j in range(1,8):
    if j==2:
        pass
    else:
        dfi["female_educ"+str(j)]=dfi["female"]*dfi["educ_"+str(j)+".0"]

for j in range(1,10):
        dfi["female_relab"+str(j)]=dfi['female']*dfi["relab_"+str(j)+".0"]

for j in range(1,8):
    if j==2:
        pass
    else:
        dfi["age_educ"+str(j)]=dfi["age"]*dfi["educ_"+str(j)+".0"]

#Create new variables
dfi["p6426_2"]=dfi["p6426"]**2
dfi["p6426_3"]=dfi["p6426"]**3
dfi["p6426_age"]=dfi["p6426"]*dfi["age"]
dfi["p6426_2_age"]=dfi["p6426_2"]*dfi["age"]
dfi["p6426_3_age"]=dfi["p6426_3"]*dfi["age"]

dfi["p7510s7a1_hours"]=dfi["p7510s7a1"]*dfi["hoursWorkUsual"]

#Prediction
#Split the data
np.random.seed(500)
dfi_train, dfi_test = train_test_split(
                                dfi,
                                test_size=.3,
                                random_state=911)

#Define X matrix
oficio=["oficio_1.0"]
for i in range(2,99):
    oficio.append("oficio_"+str(i)+".0")
    list1=['oficio_10.0', 'oficio_22.0', 'oficio_23.0', 'oficio_24.0', 'oficio_25.0', 'oficio_26.0', 'oficio_27.0', 'oficio_28.0', 'oficio_29.0', 'oficio_46.0', 'oficio_47.0', 'oficio_48.0', 'oficio_64.0', 'oficio_65.0', 'oficio_66.0', 'oficio_67.0', 'oficio_68.0', 'oficio_69.0', 'oficio_71.0']
    oficio2=[i for i in oficio if i not in list1]

x1=(dfi_train[["sizeFirm", "hoursWorkUsual", "formal", "p6426", "educ_3.0", "educ_4.0",
"educ_5.0", "educ_6.0", "educ_7.0", "relab_2.0", "relab_3.0" , "relab_4.0", "relab_5.0", "relab_6.0",
"relab_7.0", "relab_8.0" , "relab_9.0", "mes_1.0", "mes_2.0", "mes_3.0", "mes_4.0", "mes_5.0", "mes_6.0",
"mes_7.0", "mes_9.0", "mes_10.0", "mes_11.0", "mes_12.0"]])
x2=dfi_train[oficio2]
X_train=pd.concat([x1,x2], axis=1, join="inner")

x11=(dfi_test[["sizeFirm", "hoursWorkUsual", "formal", "p6426", "educ_3.0", "educ_4.0",
"educ_5.0", "educ_6.0", "educ_7.0", "relab_2.0", "relab_3.0" , "relab_4.0", "relab_5.0", "relab_6.0",
"relab_7.0", "relab_8.0" , "relab_9.0", "mes_1.0", "mes_2.0", "mes_3.0", "mes_4.0", "mes_5.0", "mes_6.0",
"mes_7.0", "mes_9.0", "mes_10.0", "mes_11.0", "mes_12.0"]])
x22=dfi_test[oficio2]
X_test=pd.concat([x11,x22], axis=1, join="inner")

rmsemodels=[0]*10

#Models
#Model 1 earnings age age2
y_train=dfi_train["log_earnings"]
y_test=dfi_test["log_earnings"]
x_train=dfi_train[["age", "age2"]]
x_test=dfi_test[["age", "age2"]]
model1=LinearRegression(fit_intercept=True)
results1= model1.fit(x_train,y_train)
y_hattest=results1.predict(x_test)

rmse1=mean_squared_error(y_test,y_hattest, squared=False)
rmse1
rmsemodels[0]=rmse1
rmse1

#Models
#Model 2 log_earnings female
y_train=dfi_train["log_earnings"]
y_test=dfi_test["log_earnings"]
x_train=dfi_train[["female"]]
x_test=dfi_test[["female"]]
model2=LinearRegression(fit_intercept=True)
results2= model2.fit(x_train,y_train)
y_hattest=results2.predict(x_test)
rmse2=mean_squared_error(y_test,y_hattest, squared=False)
rmse2
rmsemodels[1]=rmse2

#Models
#Model 3 log_earnings female age age2 agefemale agefemale
y_train=dfi_train["log_earnings"]
y_test=dfi_test["log_earnings"]
x_train=dfi_train[["female", "age", "age2", "female_age", "female_age2"]]
x_test=dfi_test[["female", "age", "age2", "female_age", "female_age2"]]
model3=LinearRegression(fit_intercept=True)
results3= model3.fit(x_train,y_train)
y_hattest=results3.predict(x_test)
rmse3=mean_squared_error(y_test,y_hattest, squared=False)
rmse3
rmsemodels[2]=rmse3

#Models
#Model 4 log_earnings female age age2 agefemale age2female X
y_train=dfi_train["log_earnings"]
y_test=dfi_test["log_earnings"]

x_trainsub=dfi_train[["female", "age", "age2", "female_age", "female_age2"]]
x_testsub=dfi_test[["female", "age", "age2", "female_age", "female_age2"]]

x_train=pd.concat([X_train,x_trainsub], axis=1, join="inner")
x_test=pd.concat([X_test,x_testsub], axis=1, join="inner")

model4=LinearRegression(fit_intercept=True)
results4= model4.fit(x_train,y_train)
y_hattest=results4.predict(x_test)
rmse4=mean_squared_error(y_test,y_hattest, squared=False)
rmse4

rmsemodels[3]=rmse4

#Models
#Model 5 age age2 hoursworkusual hoursworkusual2 agehoursworkusual agehoursworkusual2 X
y_train=dfi_train["log_earnings"]
y_test=dfi_test["log_earnings"]

x_trainsub=dfi_train[["age", "age2", "hours2", "age_hours" , "age_hours2"]]
x_testsub=dfi_test[["age", "age2", "hours2", "age_hours" , "age_hours2"]]

x_train=pd.concat([X_train,x_trainsub], axis=1, join="inner")
x_test=pd.concat([X_test,x_testsub], axis=1, join="inner")

model5=LinearRegression(fit_intercept=True)
results5= model5.fit(x_train,y_train)
y_hattest=results5.predict(x_test)
rmse5=mean_squared_error(y_test,y_hattest, squared=False)
rmse5

rmsemodels[4]=rmse5

#Models
#Model 6 age age2 female educ relab formal oficio p7090 mes reg_salud estrato
y_train=dfi_train["log_earnings"]
y_test=dfi_test["log_earnings"]

educ=["educ_1.0"]
for i in range(3,7):
    educ.append("educ_"+str(i)+".0")
x_educt=dfi_train[educ]
x_eductt=dfi_test[educ]

relab=["relab_2.0"]
for i in range(3,9):
    relab.append("relab_"+str(i)+".0")
x_relabt=dfi_train[relab]
x_relabtt=dfi_test[relab]

regs=["regs_1.0"]
for i in range(2,3):
    regs.append("regs_"+str(i)+".0")
x_regst=dfi_train[regs]
x_regstt=dfi_test[regs]

mes=["mes_1.0"]
for i in range(2,12):
    mes.append("mes_"+str(i)+".0")
x_mest=dfi_train[mes]
x_mestt=dfi_test[mes]

estrato2=["estrato_1.0"]
for i in range(2,6):
    estrato2.append("estrato_"+str(i)+".0")
x_est=dfi_train[estrato2]
x_estt=dfi_test[estrato2]

oficio=["oficio_1.0"]
for i in range(2,99):
    oficio.append("oficio_"+str(i)+".0")
    list1=['oficio_10.0', 'oficio_22.0', 'oficio_23.0', 'oficio_24.0', 'oficio_25.0', 'oficio_26.0', 'oficio_27.0', 'oficio_28.0', 'oficio_29.0', 'oficio_46.0', 'oficio_47.0', 'oficio_48.0', 'oficio_64.0', 'oficio_65.0', 'oficio_66.0', 'oficio_67.0', 'oficio_68.0', 'oficio_69.0', 'oficio_71.0']
    oficio2=[i for i in oficio if i not in list1]
x_oficiot=dfi_train[oficio2]
x_oficiott=dfi_test[oficio2]

x_trainsub=dfi_train[["age", "age2", "female" , "p7090" ]]
x_testsub=dfi_test[["age", "age2", "female" , "p7090" ]]

x_train=pd.concat([x_regst, x_est, x_educt, x_relabt, x_mest, x_oficiot,x_trainsub], axis=1, join="inner")
x_test=pd.concat([x_regstt, x_estt, x_eductt, x_relabtt, x_mestt, x_oficiott,x_testsub], axis=1, join="inner")

#Run Model 6
model6=LinearRegression(fit_intercept=True)
results6= model6.fit(x_train,y_train)
y_hattest=results6.predict(x_test)
rmse6=mean_squared_error(y_test,y_hattest, squared=False)
rmse6

rmsemodels[5]=rmse6

#Models
#Model 7 age age2 female educ femalexeduc p7090 femalexrelab relab X
y_train=dfi_train["log_earnings"]
y_test=dfi_test["log_earnings"]

female_edu=["female_educ1"]
for i in range(3,8):
    female_edu.append("female_educ"+str(i))
x_feduct=dfi_train[female_edu]
x_feductt=dfi_test[female_edu]

female_relab=["female_relab1"]
for i in range(2,10):
    female_relab.append("female_relab"+str(i))
x_ferelabt=dfi_train[female_relab]
x_ferelabtt=dfi_test[female_relab]

x_trainsub=dfi_train[["age", "age2", "female" ,"p7090"]]
x_testsub=dfi_test[["age", "age2", "female" ,"p7090"]]

x_train=pd.concat([X_train ,x_trainsub,x_feduct, x_ferelabt], axis=1, join="inner")
x_test=pd.concat([X_test,x_testsub,x_feductt, x_ferelabtt], axis=1, join="inner")

#Run Model 7
model7=LinearRegression(fit_intercept=True)
results7= model7.fit(x_train,y_train)
y_hattest=results7.predict(x_test)
rmse7=mean_squared_error(y_test,y_hattest, squared=False)
rmse7

rmsemodels[6]=rmse7

#Models
#Model 8 age age2  p7510s7a1 p7090 X p7510s7a1_hours hoursworkusual
y_train=dfi_train["log_earnings"]
y_test=dfi_test["log_earnings"]

x_trainsub=dfi_train[["age", "age2", "p7510s7a1" , "p7090", "p7510s7a1_hours" ]]
x_testsub=dfi_test[["age", "age2",  "p7510s7a1" , "p7090", "p7510s7a1_hours" ]]

x_train=pd.concat([X_train,x_trainsub], axis=1, join="inner")
x_test=pd.concat([X_test, x_testsub], axis=1, join="inner")

model8=LinearRegression(fit_intercept=True)
results8= model8.fit(x_train,y_train)
y_hattest=results8.predict(x_test)
rmse8=mean_squared_error(y_test,y_hattest, squared=False)
rmse8

rmsemodels[7]=rmse8

#Models
#Model 9 p6426 p6426_2 p6426_3 age p7090 age2 p6426Xage p6426_2age p6426_3age  X
y_train=dfi_train["log_earnings"]
y_test=dfi_test["log_earnings"]

x_trainsub=dfi_train[["p6426_2" , "p6426_3" , "age" , "age2" , "p6426_age" , "p6426_2_age" , "p6426_3_age"]]
x_testsub=dfi_test[["p6426_2" , "p6426_3" , "age" , "age2" , "p6426_age" , "p6426_2_age" , "p6426_3_age"]]

x_train=pd.concat([X_train,x_trainsub], axis=1, join="inner")
x_test=pd.concat([X_test, x_testsub], axis=1, join="inner")

model9=LinearRegression(fit_intercept=True)
results9= model9.fit(x_train,y_train)
y_hattest=results9.predict(x_test)
rmse9=mean_squared_error(y_test,y_hattest, squared=False)
rmse9

rmsemodels[8]=rmse9

#Models
#Model 10  agexeduc educ age p7090 age2 X
y_train=dfi_train["log_earnings"]
y_test=dfi_test["log_earnings"]

x_trainsub=dfi_train[["age" , "p7090" , "age2"]]
x_testsub=dfi_test[["age" , "p7090" , "age2"]]

x_train=pd.concat([X_train,x_trainsub], axis=1, join="inner")
x_test=pd.concat([X_test, x_testsub], axis=1, join="inner")

model10=LinearRegression(fit_intercept=True)
results10= model10.fit(x_train,y_train)
y_hattest=results10.predict(x_test)
rmse10=mean_squared_error(y_test,y_hattest, squared=False)
rmse10

rmsemodels[9]=rmse10

#Influence statistic
#Model 5
y_train=dfi_train["log_earnings"]
y_test=dfi_test["log_earnings"]

x_trainsub=dfi_train[["age", "age2", "hours2", "age_hours" , "age_hours2"]]
x_testsub=dfi_test[["age", "age2",  "hours2", "age_hours" , "age_hours2"]]

x_train=pd.concat([X_train,x_trainsub], axis=1, join="inner")
x_test=pd.concat([X_test,x_testsub], axis=1, join="inner")

#compute diagonal of PX
olsmod= sm.OLS(y_test, x_test)
olsmodfit=olsmod.fit()
a=OLSInfluence(olsmodfit)
h=a.hat_matrix_diag

#Residuals in test
olsmod2= sm.OLS(y_train, x_train)
olsmodfit2=olsmod2.fit()
y_hattest2=olsmodfit2.predict(x_test)
residuals2=y_test-y_hattest

residuals2=np.array(residuals2)

#Create the influence for all j in test
influence=[0.0]*4880
influence=np.array(influence)
influence

for j in range(0,4880):
    influence[j]=residuals2[j]/(1-h[j])

plt.xlabel('Valor del estadistico de influencia')
plt.ylabel('Frecuencia')
n, bins, patches = plt.hist(influence , bins=100, range=[0,1] ,facecolor='g', alpha=0.2)
plt.savefig('influencia.png')

#CrossValidation
#Model 5 age age2 hoursworkusual hoursworkusual2 agehoursworkusual agehoursworkusual2 X
#Define Cross-validation variables
y_cv=dfi["log_earnings"]
x_cvsub=dfi[["age", "age2", "hours2", "age_hours" , "age_hours2"]]

oficio=["oficio_1.0"]
for i in range(2,99):
    oficio.append("oficio_"+str(i)+".0")
    list1=['oficio_10.0', 'oficio_22.0', 'oficio_23.0', 'oficio_24.0', 'oficio_25.0', 'oficio_26.0', 'oficio_27.0', 'oficio_28.0', 'oficio_29.0', 'oficio_46.0', 'oficio_47.0', 'oficio_48.0', 'oficio_64.0', 'oficio_65.0', 'oficio_66.0', 'oficio_67.0', 'oficio_68.0', 'oficio_69.0', 'oficio_71.0']
    oficio2=[i for i in oficio if i not in list1]

x1=(dfi[["sizeFirm", "hoursWorkUsual", "formal", "p6426", "educ_3.0", "educ_4.0",
"educ_5.0", "educ_6.0", "educ_7.0", "relab_2.0", "relab_3.0" , "relab_4.0", "relab_5.0", "relab_6.0",
"relab_7.0", "relab_8.0" , "relab_9.0", "mes_1.0", "mes_2.0", "mes_3.0", "mes_4.0", "mes_5.0", "mes_6.0",
"mes_7.0", "mes_9.0", "mes_10.0", "mes_11.0", "mes_12.0"]])
x2=dfi[oficio2]
x_cv0=pd.concat([x1,x2], axis=1, join="inner")

x_cv=pd.concat([x_cv0,x_cvsub], axis=1, join="inner")

#CrossValidation of Model 5 ###############################3
#define cross-validation method to use
cv = LeaveOneOut()
#Define cross-validation model
model5=LinearRegression(fit_intercept=True)
#use LOOCV to evaluate model
scores = cross_val_score(model5, x_cv, y_cv, scoring='neg_root_mean_squared_error',
                         cv=cv, n_jobs=-1)

rMSE5 = np.mean(np.absolute(scores))

#CrossValidation Model 7 age age2 female educ femalexeduc p7090 femalexrelab relab X
#Define cross-validation variables
y_cv2=dfi["log_earnings"]

female_edu=["female_educ1"]
for i in range(3,8):
    female_edu.append("female_educ"+str(i))
x_feduct=dfi[female_edu]

female_relab=["female_relab1"]
for i in range(2,10):
    female_relab.append("female_relab"+str(i))
x_ferelabt=dfi[female_relab]

x_cv2sub=dfi[["age", "age2", "female" ,"p7090"]]

oficio=["oficio_1.0"]
for i in range(2,99):
    oficio.append("oficio_"+str(i)+".0")
    list1=['oficio_10.0', 'oficio_22.0', 'oficio_23.0', 'oficio_24.0', 'oficio_25.0', 'oficio_26.0', 'oficio_27.0', 'oficio_28.0', 'oficio_29.0', 'oficio_46.0', 'oficio_47.0', 'oficio_48.0', 'oficio_64.0', 'oficio_65.0', 'oficio_66.0', 'oficio_67.0', 'oficio_68.0', 'oficio_69.0', 'oficio_71.0']
    oficio2=[i for i in oficio if i not in list1]

x1=(dfi[["sizeFirm", "hoursWorkUsual", "formal", "p6426", "educ_3.0", "educ_4.0",
"educ_5.0", "educ_6.0", "educ_7.0", "relab_2.0", "relab_3.0" , "relab_4.0", "relab_5.0", "relab_6.0",
"relab_7.0", "relab_8.0" , "relab_9.0", "mes_1.0", "mes_2.0", "mes_3.0", "mes_4.0", "mes_5.0", "mes_6.0",
"mes_7.0", "mes_9.0", "mes_10.0", "mes_11.0", "mes_12.0"]])
x2=dfi[oficio2]
x_cv02=pd.concat([x1,x2], axis=1, join="inner")

x_cv2=pd.concat([x_cv2sub,x_feduct, x_ferelabt, x_cv02 ], axis=1, join="inner")

#CrossValidation of Model 7 ###############################3
#define cross-validation method to use
cv = LeaveOneOut()
#Define cross-validation model
model7=LinearRegression(fit_intercept=True)
#use LOOCV to evaluate model
scores2 = cross_val_score(model7, x_cv2, y_cv2, scoring='neg_root_mean_squared_error',
                         cv=cv, n_jobs=-1)

#view RMSE
rMSE7 = np.mean(np.absolute(scores2))


import torch
import torch.distributions as dist
from escnn import gspaces

from models.model import SteerableCNN
from utils import BatchGeometricTensor
from common.blocks import EmbeddingBlock

class SteerableCNN_QM9(SteerableCNN):
    def __init__(self, gspace: gspaces, filter_type: str, non_linearity: str,
                 n_g_blocks: int, num_nodes: list, num_classes: int,
                 connectivity: str, channels: list, max_freq: list, node_dim: int, edge_dim: int, edge_att_type: str,
                 use_bn: int, x_dist: dist = None, attr_dist: dist = None, pool_type: str = 'mean',
                 use_skipcons: bool = False, aggr: str = 'add', **kwargs):

        embed_block = EmbeddingBlock(
            non_linearity=non_linearity,
            channels=channels[0],
            L=max_freq[0],
            in_type=gspace.type(*(node_dim * [gspace.trivial_repr]))
        )

        super().__init__(embed_block, filter_type, non_linearity, n_g_blocks, num_nodes, num_classes,
                 connectivity, channels, max_freq, edge_dim, edge_att_type, use_bn,
                 x_dist, attr_dist, pool_type, use_skipcons, aggr, **kwargs)

    def forward(self, pos_in: torch.Tensor, batch: torch.Tensor,
                node_features: torch.Tensor = None, edge_index: torch.Tensor = None, edge_attr: torch.Tensor = None):
        """
        Forward pass of the network.

        Args:
            pos_in (torch.Tensor): Input positions of shape `(num_nodes, 3)`.
            batch (torch.Tensor): Batch vector of shape `(num_nodes, )`.
            node_features (torch.Tensor, optional): Node features of shape `(num_nodes, num_node_features)`.
                Defaults to None.
            edge_index (torch.Tensor, optional): Edge indices of shape `(2, num_edges)` for graph connectivity.
                Defaults to None.
            edge_attr (torch.Tensor, optional): Edge attributes of shape `(num_edges, num_edge_features)`.
                Defaults to None.

        Returns:
            torch.Tensor: Output tensor of shape `(num_samples, num_classes)`.
        """
        x = self.embed_block(node_features, pos_in)

        # induce connectivity if 'knn' or 'radius' passed
        if self.connectivity != "given":
            edge_index = None

        x = BatchGeometricTensor(
            x,
            batch,
            node_features=node_features,
            edge_index=edge_index,
            edge_attr=edge_attr,
        )
        x = self.g_blocks(x)
        x, batch = self.inv_block(x)
        x = self.pooling(x, batch)
        x = self.fully_net(x)

        return x


import UNet.Up as Up
import UNet.Down as Down
import torch.nn as nn

class Unet(nn.Module):

    def __init__(self, down_channels = [1, 2], up_channels = [2, 1], retain_size=True):
        super().__init__()
        self.down_arc = Down.Down(down_channels)
        self.up_arc = Up.Up(up_channels)
        self.retain_size = retain_size
        self.result = nn.Conv2d(up_channels[-1], 4, 1)

    def forward(self, x, out_size=(512,512)):
        down_features = self.down_arc(x)
        res = self.up_arc(down_features[::-1][0], down_features[::-1][1:])
        res = self.result(res)
        if self.retain_size:
            res = nn.functional.interpolate(res, out_size)
        return res


#!/usr/bin/env python

# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
import csv
import json
import os
import re
import shutil
import subprocess
import sys

from talos.xtalos import xtalos

EVENTNAME_INDEX = 0
PROCESS_INDEX = 2
THREAD_ID_INDEX = 3
PARENT_PID_INDEX = 3
DISKBYTES_COL = "Size"
FNAME_COL = "FileName"
IMAGEFUNC_COL = "Image!Function"
EVENTGUID_COL = "EventGuid"
ACTIVITY_ID_COL = "etw:ActivityId"
NUMBYTES_COL = "NumBytes"
BYTESSENT_COL = "BytesSent"

CEVT_WINDOWS_RESTORED = "{917b96b1-ecad-4dab-a760-8d49027748ae}"
CEVT_XPCOM_SHUTDOWN = "{26d1e091-0ae7-4f49-a554-4214445c505c}"
NAME_SUBSTITUTIONS = [
    # Careful with your regex!
    # Substitution happens after combinations like \t \s \n ... are replaced
    # with their real representations. So, prepend them with extra backslash.
    # Read more: https://docs.python.org/2.7/library/re.html#re.sub
    (re.compile(r"{\w{8}-\w{4}-\w{4}-\w{4}-\w{12}}"), "{uuid}"),
    (re.compile(r"talos\\tests\\tp5n\\.*"), r"talos\\tests\{tp5n_files}"),
    (re.compile(r"nvidia corporation\\3d vision\\.*"), "{nvidia_3d_vision}"),
    (re.compile(r"cltbld\.t-w732-ix-\d+\.\d+"), "{cltbld}"),
    (
        re.compile(r"venv\\lib\\site-packages\\pip\\_vendor\\.*"),
        r"venv\\lib\\site-packages\\{pip_vendor}",
    ),
]
stages = ["startup", "normal", "shutdown"]
net_events = {
    "TcpDataTransferReceive": "recv",
    "UdpEndpointReceiveMessages": "recv",
    "TcpDataTransferSend": "send",
    "UdpEndpointSendMessages": "send",
}
gThreads = {}
gConnectionIDs = {}
gHeaders = {}
gBrowserPID = None

def uploadFile(filename):
    mud = os.environ.get("MOZ_UPLOAD_DIR", None)
    if mud:
        print("uploading raw file %s via blobber" % filename)
        mud_filename = os.path.join(mud, filename)
        shutil.copyfile(filename, "%s.log" % mud_filename)

def filterOutHeader(data):
    # -1 means we have not yet found the header
    # 0 means we are in the header
    # 1+ means that we are past the header
    state = -1
    done = False
    while not done:
        try:
            row = next(data)
        except StopIteration:
            done = True
            break
        except UnicodeDecodeError:
            continue
        except csv.Error:
            continue

        if not len(row):
            continue

        if state < 0:
            # Keep looking for the header (denoted by "BeginHeader").
            if row[0] == "BeginHeader":
                state = 0
            continue

        if state == 0:
            # Eventually, we'll find the end (denoted by "EndHeader").
            if row[0] == "EndHeader":
                state = 1
                continue

            gHeaders[row[EVENTNAME_INDEX]] = row
            continue

        if state >= 1:
            state = state + 1

        # The line after "EndHeader" is also not useful, so we want to strip
        # that in addition to the header.
        if state > 2:
            yield row

def getIndex(eventName, *colNames):
    eventHeader = gHeaders[eventName]

    for colName in colNames:
        try:
            return eventHeader.index(colName)
        except ValueError:
            pass

    return None

def readFile(filename):
    data = csv.reader(
        open(filename, "r", encoding="cp1252"),
        delimiter=",",
        quotechar='"',
        skipinitialspace=True,
    )
    data = filterOutHeader(data)
    return data

def fileSummary(row, stage, retVal):
    event = row[EVENTNAME_INDEX]

    # TODO: do we care about the other events?
    if event not in ("FileIoRead", "FileIoWrite"):
        return
    fname_index = getIndex(event, FNAME_COL)

    # We only care about events that have a file name.
    if fname_index is None:
        return

    # Some data rows are missing the filename?
    if len(row) <= fname_index:
        return

    thread_extra = ""
    if gThreads[row[THREAD_ID_INDEX]] == "main":
        thread_extra = " (main)"
    key_tuple = (
        row[fname_index],
        "%s%s" % (row[THREAD_ID_INDEX], thread_extra),
        stages[stage],
    )
    total_tuple = (
        row[fname_index],
        "%s%s" % (row[THREAD_ID_INDEX], thread_extra),
        "all",
    )

    if key_tuple not in retVal:
        retVal[key_tuple] = {
            "DiskReadBytes": 0,
            "DiskReadCount": 0,
            "DiskWriteBytes": 0,
            "DiskWriteCount": 0,
        }
        retVal[total_tuple] = retVal[key_tuple]

    if event == "FileIoRead":
        retVal[key_tuple]["DiskReadCount"] += 1
        retVal[total_tuple]["DiskReadCount"] += 1
        idx = getIndex(event, DISKBYTES_COL)
        retVal[key_tuple]["DiskReadBytes"] += int(row[idx], 16)
        retVal[total_tuple]["DiskReadBytes"] += int(row[idx], 16)
    elif event == "FileIoWrite":
        retVal[key_tuple]["DiskWriteCount"] += 1
        retVal[total_tuple]["DiskWriteCount"] += 1
        idx = getIndex(event, DISKBYTES_COL)
        retVal[key_tuple]["DiskWriteBytes"] += int(row[idx], 16)
        retVal[total_tuple]["DiskWriteBytes"] += int(row[idx], 16)

def etl2csv(xperf_path, etl_filename, debug=False):
    """
    Convert etl_filename to etl_filename.csv (temp file) which is the .csv
    representation of the .etl file Etlparser will read this .csv and parse
    the information we care about into the final output. This is done to keep
    things simple and to preserve resources on talos machines
    (large files == high memory + cpu)
    """

    xperf_cmd = [
        xperf_path,
        "-merge",
        "%s.user" % etl_filename,
        "%s.kernel" % etl_filename,
        etl_filename,
    ]
    if debug:
        print("executing '%s'" % subprocess.list2cmdline(xperf_cmd))
    subprocess.call(xperf_cmd)

    csv_filename = "%s.csv" % etl_filename
    xperf_cmd = [xperf_path, "-i", etl_filename, "-o", csv_filename]
    if debug:
        print("executing '%s'" % subprocess.list2cmdline(xperf_cmd))
    subprocess.call(xperf_cmd)
    return csv_filename

def trackProcess(row, firstFirefoxPID):
    global gBrowserPID
    if gBrowserPID:
        return

    # Without the launcher, the initial Firefox process *is* the browser
    # process. OTOH, with the launcher process enabled, the browser is actually
    # the first child process of the first Firefox process.
    parentPID = int(row[PARENT_PID_INDEX])
    if parentPID == firstFirefoxPID:
        proc = row[PROCESS_INDEX]
        gBrowserPID = int(re.search("^.* \(\s*(\d+)\)$", proc).group(1))

def getBrowserPID():
    global gBrowserPID
    return gBrowserPID

def trackThread(row, browserPID):
    event, proc, tid = row[EVENTNAME_INDEX], row[PROCESS_INDEX], row[THREAD_ID_INDEX]
    if event in ["T-DCStart", "T-Start"]:
        procName, procID = re.search("^(.*) \(\s*(\d+)\)$", proc).group(1, 2)
        if procID == str(browserPID):
            imgIdx = getIndex(event, IMAGEFUNC_COL)
            img = re.match("([^!]+)!", row[imgIdx]).group(1)
            if img == procName:
                gThreads[tid] = "main"
            else:
                gThreads[tid] = "nonmain"
    elif event in ["T-DCEnd", "T-End"] and tid in gThreads:
        del gThreads[tid]

def trackThreadFileIO(row, io, stage):
    event, tid = row[EVENTNAME_INDEX], row[THREAD_ID_INDEX]
    opType = {"FileIoWrite": "write", "FileIoRead": "read"}[event]
    th, stg = gThreads[tid], stages[stage]
    sizeIdx = getIndex(event, DISKBYTES_COL)
    bytes = int(row[sizeIdx], 16)
    io[(th, stg, "file_%s_ops" % opType)] = (
        io.get((th, stg, "file_%s_ops" % opType), 0) + 1
    )
    io[(th, stg, "file_%s_bytes" % opType)] = (
        io.get((th, stg, "file_%s_bytes" % opType), 0) + bytes
    )
    io[(th, stg, "file_io_bytes")] = io.get((th, stg, "file_io_bytes"), 0) + bytes

def trackThreadNetIO(row, io, stage):
    event, tid = row[EVENTNAME_INDEX], row[THREAD_ID_INDEX]
    connIdIdx = getIndex(event, ACTIVITY_ID_COL)
    connID = row[connIdIdx]
    if connID not in gConnectionIDs:
        gConnectionIDs[connID] = tid
    origThread = gConnectionIDs[connID]
    if origThread in gThreads:
        match = re.match("[\w-]+\/([\w-]+)?", event)
        if not match:
            raise xtalos.XTalosError(
                "Could not find a regular expression match for event: {}".format(event)
            )
        netEvt = match.group(1)

        if netEvt in net_events:
            opType = net_events[netEvt]
            th, stg = gThreads[origThread], stages[stage]
            # On newer versions of Windows, some net I/O events have switched to
            # using BYTESSENT_COL, so we try both
            lenIdx = getIndex(event, NUMBYTES_COL, BYTESSENT_COL)
            bytes = int(row[lenIdx])
            io[(th, stg, "net_%s_bytes" % opType)] = (
                io.get((th, stg, "net_%s_bytes" % opType), 0) + bytes
            )
            io[(th, stg, "net_io_bytes")] = io.get((th, stg, "net_io_bytes"), 0) + bytes

def updateStage(row, stage):
    guidIdx = getIndex(row[EVENTNAME_INDEX], EVENTGUID_COL)
    if row[guidIdx] == CEVT_WINDOWS_RESTORED and stage == 0:
        stage = 1
    elif row[guidIdx] == CEVT_XPCOM_SHUTDOWN and stage == 1:
        stage = 2
    return stage

def loadAllowlist(filename):
    if not filename:
        return
    if not os.path.exists(filename):
        print("Warning: xperf allowlist %s was not found" % filename)
        return
    lines = open(filename).readlines()
    # Expand paths
    lines = [os.path.expandvars(elem.strip()) for elem in lines]
    files = set()
    dirs = set()
    recur = set()
    for line in lines:
        if line.startswith("#"):
            continue
        elif line.endswith("\\*\\*"):
            recur.add(line[:-4])
        elif line.endswith("\\*"):
            dirs.add(line[:-2])
        else:
            files.add(line)
    return (files, dirs, recur)

def checkAllowlist(filename, allowlist):
    if not allowlist:
        return False
    if filename in allowlist[0]:
        return True
    if os.path.dirname(filename) in allowlist[1]:
        return True
    head = filename
    while len(head) > 3:  # Length 3 implies root directory, e.g. C:\
        head, tail = os.path.split(head)
        if head in allowlist[2]:
            return True
    return False

def etlparser(
    xperf_path,
    etl_filename,
    processID,
    approot=None,
    configFile=None,
    outputFile=None,
    allowlist_file=None,
    error_filename=None,
    all_stages=False,
    all_threads=False,
    debug=False,
):
    global NAME_SUBSTITUTIONS

    # setup output file
    if outputFile:
        outFile = open(outputFile, "w")
    else:
        outFile = sys.stdout

    files = {}
    io = {}
    stage = 0

    csvname = etl2csv(xperf_path, etl_filename, debug=debug)
    for row in readFile(csvname):
        event = row[EVENTNAME_INDEX]
        if event == "P-Start":
            trackProcess(row, processID)
        elif event in ["T-DCStart", "T-Start", "T-DCEnd", "T-End"]:
            trackThread(row, getBrowserPID())
        elif (
            event in ["FileIoRead", "FileIoWrite"] and row[THREAD_ID_INDEX] in gThreads
        ):
            fileSummary(row, stage, files)
            trackThreadFileIO(row, io, stage)
        elif event.endswith("Event/Classic") and row[THREAD_ID_INDEX] in gThreads:
            stage = updateStage(row, stage)
        elif event.startswith("Microsoft-Windows-TCPIP"):
            trackThreadNetIO(row, io, stage)

    if debug:
        uploadFile(csvname)

    output = "thread, stage, counter, value\n"
    for cntr in sorted(io.keys()):
        output += "%s, %s\n" % (", ".join(cntr), str(io[cntr]))
    if outputFile:
        fname = "%s_thread_stats%s" % os.path.splitext(outputFile)
        with open(fname, "w") as f:
            f.write(output)

        if debug:
            uploadFile(fname)
    else:
        print(output)

    allowlist = loadAllowlist(allowlist_file)

    header = "filename, tid, stage, readcount, readbytes, writecount," " writebytes"
    outFile.write(header + "\n")

    # Filter out stages, threads, and allowlisted files that we're not
    # interested in
    filekeys = [
        x
        for x in files.keys()
        if (all_stages or x[2] == stages[0])
        and (all_threads or x[1].endswith("(main)"))
        and (all_stages and x[2] != stages[0] or not checkAllowlist(x[0], allowlist))
    ]
    if debug:
        # in debug, we want stages = [startup+normal] and all threads, not just (main)
        # we will use this data to upload fileIO info to blobber only for debug mode
        outputData = [
            x
            for x in files.keys()
            if (all_stages or x[2] in [stages[0], stages[1]])
            and (
                all_stages
                and x[2] not in [stages[0], stages[1]]
                or not checkAllowlist(x[0], allowlist)
            )
        ]
    else:
        outputData = filekeys

    # output data
    for row in outputData:
        output = "%s, %s, %s, %s, %s, %s, %s\n" % (
            row[0],
            row[1],
            row[2],
            files[row]["DiskReadCount"],
            files[row]["DiskReadBytes"],
            files[row]["DiskWriteCount"],
            files[row]["DiskWriteBytes"],
        )
        outFile.write(output)

    if outputFile:
        # close the file handle
        outFile.close()
        if debug:
            uploadFile(outputFile)

    # We still like to have the outputfile to record the raw data, now
    # filter out acceptable files/ranges
    allowlist_path = None
    wl_temp = {}
    dirname = os.path.dirname(__file__)
    if os.path.exists(os.path.join(dirname, "xperf_allowlist.json")):
        allowlist_path = os.path.join(dirname, "xperf_allowlist.json")
    elif os.path.exists(os.path.join(dirname, "xtalos")) and os.path.exists(
        os.path.join(dirname, "xtalos", "xperf_allowlist.json")
    ):
        allowlist_path = os.path.join(dirname, "xtalos", "xperf_allowlist.json")

    wl_temp = {}
    if allowlist_path:
        with open(allowlist_path, "r") as fHandle:
            wl_temp = json.load(fHandle)

    # Approot is the full path where the application is located at
    # We depend on it for dependentlibs.list to ignore files required for
    # normal startup.
    if approot:
        if os.path.exists("%s\\dependentlibs.list" % approot):
            with open("%s\\dependentlibs.list" % approot, "r") as fhandle:
                libs = fhandle.readlines()

            for lib in libs:
                wl_temp["{firefox}\\%s" % lib.strip()] = {"ignore": True}

    # Windows isn't case sensitive, this protects us against mismatched
    # systems.
    wl = {}
    for item in wl_temp:
        wl[item.lower()] = wl_temp[item]

    errors = []
    for row in filekeys:
        filename = original_filename = row[0]
        filename = filename.lower()
        # take care of 'program files (x86)' matching 'program files'
        filename = filename.replace(" (x86)", "")

        paths = ["profile", "firefox", "desktop", "talos"]
        for path in paths:
            pathname = "%s\\" % path
            parts = filename.split(pathname)
            if len(parts) >= 2:
                filename = "{%s}\\%s" % (path, pathname.join(parts[1:]))

        parts = filename.split("\\installtime")
        if len(parts) >= 2:
            filename = "%s\\{time}" % parts[0]

        # NOTE: this is Prefetch or prefetch, not case sensitive operating
        # system
        parts = filename.split("refetch")
        if len(parts) >= 2:
            filename = "%srefetch\\{prefetch}.pf" % parts[0]

        for pattern, substitution in NAME_SUBSTITUTIONS:
            filename = re.sub(pattern, substitution, filename)

        if filename in wl:
            if "ignore" in wl[filename] and wl[filename]["ignore"]:
                continue

        # too noisy
        #            if wl[filename]['minbytes'] > (files[row]['DiskReadBytes'] +\
        #                   files[row]['DiskWriteBytes']):
        #                print "%s: read %s bytes, less than expected minimum: %s"
        #                       % (filename, (files[row]['DiskReadBytes'] +
        #                                     files[row]['DiskWriteBytes']),
        #                          wl[filename]['minbytes'])

        # don't report in first round
        #            elif wl[filename]['maxbytes'] < (files[row]['DiskReadBytes'] +\
        #                   files[row]['DiskWriteBytes']):
        #                errors.append("%s: read %s bytes, more than expected maximum:"
        #                              " %s"
        #                              % (filename, (files[row]['DiskReadBytes'] +
        #                                 files[row]['DiskWriteBytes']),
        #                              wl[filename]['maxbytes']))

        # too noisy
        #            elif wl[filename]['mincount'] > (files[row]['DiskReadCount'] +\
        #                   files[row]['DiskWriteCount']):
        #                print "%s: %s accesses, less than expected minimum: %s"
        #                       % (filename, (files[row]['DiskReadCount'] +
        #                                     files[row]['DiskWriteCount']),
        #                          wl[filename]['mincount'])

        # don't report in first round
        #            elif wl[filename]['maxcount'] < (files[row]['DiskReadCount'] +\
        #                   files[row]['DiskWriteCount']):
        #                errors.append("%s: %s accesses, more than expected maximum:"
        #                              " %s"
        #                              % (filename, (files[row]['DiskReadCount'] +
        #                                            files[row]['DiskWriteCount']),
        #                                 wl[filename]['maxcount']))
        else:
            errors.append(
                "File '%s' (normalized from '%s') was accessed and we were not expecting"
                " it.  DiskReadCount: %s, DiskWriteCount: %s,"
                " DiskReadBytes: %s, DiskWriteBytes: %s"
                % (
                    filename,
                    original_filename,
                    files[row]["DiskReadCount"],
                    files[row]["DiskWriteCount"],
                    files[row]["DiskReadBytes"],
                    files[row]["DiskWriteBytes"],
                )
            )

    if errors:
        # output specific errors to be picked up by tbpl parser
        for error in errors:
            # NOTE: the ' :' is intentional, without the space before the :,
            # some parser will translate this
            print("TEST-UNEXPECTED-FAIL : xperf: %s" % error)

        # We detect if browser_failures.txt exists to exit and turn the job
        # orange
        if error_filename:
            with open(error_filename, "w") as errorFile:
                errorFile.write("\n".join(errors))

        if debug:
            uploadFile(etl_filename)

def etlparser_from_config(config_file, **kwargs):
    """start from a YAML config file"""

    # option defaults
    args = {
        "xperf_path": "xperf.exe",
        "etl_filename": "test.etl",
        "outputFile": "etl_output.csv",
        "processID": None,
        "approot": None,
        "allowlist_file": None,
        "error_filename": None,
        "all_stages": False,
        "all_threads": False,
    }
    args.update(kwargs)

    # override from YAML config file
    args = xtalos.options_from_config(args, config_file)

    # ensure process ID is given
    if not args.get("processID"):
        raise xtalos.XTalosError("No process ID option given")

    # ensure path to xperf given
    if not os.path.exists(args["xperf_path"]):
        raise xtalos.XTalosError(
            "ERROR: xperf_path '%s' does not exist" % args["xperf_path"]
        )

    # update args with config file
    args["configFile"] = config_file

    # call etlparser
    etlparser(**args)

def main(args=sys.argv[1:]):

    # parse command line arguments
    parser = xtalos.XtalosOptions()
    args = parser.parse_args(args)
    args = parser.verifyOptions(args)
    if args is None:
        parser.error("Unable to verify arguments")
    if not args.processID:
        parser.error("No process ID argument given")

    # call API
    etlparser(
        args.xperf_path,
        args.etl_filename,
        args.processID,
        args.approot,
        args.configFile,
        args.outputFile,
        args.allowlist_file,
        args.error_filename,
        args.all_stages,
        args.all_threads,
        debug=args.debug_level >= xtalos.DEBUG_INFO,
    )

if __name__ == "__main__":
    main()


import numpy as np
import pandas as pd
import altair as alt
from tqdm import tqdm
import os

def create_invert_map(lst):
    return {v: i for i, v in enumerate(lst)}

a1 = [1,2,3,7,5]
a2 = ["2","5","A"]
a3 = [False,True]
b1, b2, b3 = map(create_invert_map, [a1, a2, a3])
#print(b1.get(5))
#print(b2.get("5"))
#print(b3.get(False))
#print(b2["5"])
#print(b1)
#print(b1[7])

cards = ["A",2, 3, 4, 5, 6, 7, 8, 9, 10, 10, 10, 10]
#returns char because we have "A". If it wasn't in the list, it would return int.
sampled_cards = np.random.choice(cards, size=1, replace=True)
#print(sampled_cards)

class BlackJack:
    cards = ["A", 2, 3, 4, 5, 6, 7, 8, 9, 10, 10, 10, 10]
    agent_sums = [12, 13, 14, 15, 16, 17, 18, 19, 20, 21]
    dealers_cards = ["A", 2, 3, 4, 5, 6, 7, 8, 9, 10]
    ace_usability = [False, True]
    actions_possible = ["S", "H"]

    agent_sums_map, dealers_cards_map, ace_usability_map, actions_map, cards_map = map(
        create_invert_map,
        [agent_sums, dealers_cards, ace_usability, actions_possible, cards],
    )

    def __init__(self, M, epsilon, alpha, seed=0):
        self.seed = seed
        self.M = M #how many simulations
        self.epsilon = epsilon
        self.alpha = alpha
        self.Q_hist = None
        self.initialize()

    def initialize(self):
        # Q is a 4D array giving the state-action values.
        self.Q = np.zeros(
            (
                len(self.agent_sums),
                len(self.dealers_cards),
                len(self.ace_usability),
                len(self.actions_possible),
            )
        )
        # C is a 4D array counting the times a state-action pair was visited. It's not actually needed for the algorithm.
        self.C = np.zeros_like(self.Q, dtype=int)

        # Create the data directory if it doesn't already exist.
        if not os.path.exists('data'):
            os.mkdir('data')

    def sample_cards(self, N):
        """Draws N samples for the infinite 13 card deck."""
        sampled_cards = np.random.choice(self.cards, size=N, replace=True)
        return [c if c == "A" else int(c) for c in sampled_cards]

    def map_to_indices(self, agent_sum, dealers_card, useable_ace):
        """Map from agent_sum (12-21), dealers_card(ace-10) and useable_ace (True, False) to the indices of Q."""
        return (
            self.agent_sums_map[agent_sum],
            self.dealers_cards_map[dealers_card],
            self.ace_usability_map[useable_ace],
        )

    def behavior_policy(self, agent_sum, dealers_card, useable_ace):
        """Returns H (Hit) or S (Stick) to determine the actions to take during the game."""

        agent_sum_idx, dealers_card_idx, useable_ace_idx = self.map_to_indices(
            agent_sum, dealers_card, useable_ace
        )
        greedy_action = self.Q[
            agent_sum_idx, dealers_card_idx, useable_ace_idx, :
        ].argmax()
        do_greedy = np.random.binomial(1, 1 - self.epsilon + (self.epsilon / 2))
        if do_greedy:
            return self.actions_possible[int(greedy_action)]
        else:
            return self.actions_possible[int(not greedy_action)]

    def target_policy(self, agent_sum, dealers_card, useable_ace):
        """
        The target policy is the same as the behavior policy in on-policy learning. Note: this method is never actually
        referenced.
        """
        return self.behavior_policy(
            agent_sum=agent_sum, dealers_card=dealers_card, useable_ace=useable_ace
        )

    def is_ratio(self, states_remaining, actions_remaining):
        """The Importance Sampling ratio that can be overwritten for off policy control."""
        return 1

    @staticmethod
    def calc_sum_useable_ace(cards):
        """Returns the sum-value of cards and whether there is a useable ace"""
        cards_sum = sum([c for c in cards if c != "A"])
        ace_count = len([c for c in cards if c == "A"])

        if ace_count == 0:
            return cards_sum, False
        else:
            cards_sum_0 = cards_sum + ace_count  # 0 aces count as 11
            cards_sum_1 = cards_sum + 10 + ace_count  # 1 ace counts as 11

            if cards_sum_1 > 21:
                return cards_sum_0, False

            return cards_sum_1, True

    def play_game(self):
        """
        Returns:
            states: a list of states, one for each time state.
            actions: a list of agent actions, one for each time state.
            reward: final return of the game. Either -1, 0, 1
            how: string describing how the game ended.
        """

        agent_cards = self.sample_cards(2)  # Agent is delt 2 cards. Result is [x,y]
        dealers_card = self.sample_cards(1)[0]  # Dealer is (effectively) delt 1 cards. With [0] we get x

        states = [[agent_cards, dealers_card]] #states[0] will return first move data. state[1] second one. states[0][0] will return my hand
        actions = []

        # Hit until agent_sum >= 12
        while True:
            agent_cards, dealers_card = states[-1]# Takes the last state. From [[[10, 4], 10]] to [[10, 4], 10]

            agent_sum, _ = self.calc_sum_useable_ace(agent_cards)
            if agent_sum < 12:
                actions.append("H")
                agent_cards_next = agent_cards + self.sample_cards(1)
                states.append([agent_cards_next, dealers_card])
            else:
                break

        # Play game according to agents policy
        while True:
            agent_cards, dealers_card = states[-1]
            agent_sum, useable_ace = self.calc_sum_useable_ace(agent_cards)

            action = self.behavior_policy(
                agent_sum=agent_sum, dealers_card=dealers_card, useable_ace=useable_ace
            )
            actions.append(action)
            if action == "S":
                states.append([agent_cards, dealers_card])
                break
            else:
                agent_cards_next = agent_cards + self.sample_cards(1)
                states.append([agent_cards_next, dealers_card])

                agent_sum_next, useable_ace = self.calc_sum_useable_ace(
                    agent_cards_next
                )

                if agent_sum_next > 21:
                    how = "bust"
                    reward = -1
                    return states, actions, reward, how

        # Dealer plays
        dealers_cards = [states[-1][1]] + self.sample_cards(1)  # Turn over card
        while True:
            dealers_sum, _ = self.calc_sum_useable_ace(dealers_cards)
            if dealers_sum > 21:
                how = "dealer_bust:" + ",".join([str(c) for c in dealers_cards])
                reward = 1
                return states, actions, reward, how
            if dealers_sum > 16:
                break
            else:
                dealers_cards += self.sample_cards(1)

        agent_sum, useable_ace = self.calc_sum_useable_ace(states[-1][0])

        # If the game hasn't already ended, determine the winner.
        if agent_sum == dealers_sum:
            return states, actions, 0, f"{agent_sum} = {dealers_sum}"
        elif agent_sum > dealers_sum:
            return states, actions, 1, f"{agent_sum} > {dealers_sum}"
        else:
            return states, actions, -1, f"{agent_sum} < {dealers_sum}"

    def get_hyper_str(self):
        """Returns a string uniquely identifying the class arguments."""
        return f"M{self.M}__epsilon{str(self.epsilon).replace('.', '_')}__alpha{str(self.alpha).replace('.', '_')}__seed{str(self.seed)}"

    def get_file_names(self):
        hyper_str = self.get_hyper_str()
        Q_name = "data\\Q_" + hyper_str
        C_name = "data\\C_" + hyper_str
        Q_hist_name = "data\\Q_hist_" + hyper_str
        return Q_name, C_name, Q_hist_name

    def save(self):
        Q_name, C_name, Q_hist_name = self.get_file_names()
        print(f"Saving {Q_name}")
        np.save(Q_name, self.Q)
        print(f"Saving {C_name}")
        np.save(C_name, self.C)
        if self.Q_hist is not None:
            print(f"Saving {Q_hist_name}")
            self.Q_hist.to_pickle(Q_hist_name + ".pkl")

    def load(self):
        Q_name, C_name, Q_hist_name = self.get_file_names()
        print(f"Loading...")
        self.Q = np.load(Q_name + ".npy")
        self.C = np.load(C_name + ".npy")
        try:
            self.Q_hist = pd.read_pickle(Q_hist_name + ".pkl")
        except:
            print("No Q hist to load")

    def _get_ms(self):
        """Returns a list of episodes index's (sometimes called 'm's) for which we record action-value pairs."""
        return list(range(0, self.M + 1, 1000))

    def mc_control(self, track_history=True):
        """Return the algorithm to learn the Q-table."""

        self.initialize()
        np.random.seed(self.seed)

        Q_hist = []
        ms = self._get_ms()

        for m in tqdm(range(self.M + 1)):
            states, actions, reward, how = self.play_game()

            if track_history:
                if m in ms:
                    Q_hist.append(self.get_df("Q").assign(m=m))

            for i, (state, action) in enumerate(zip(states[:-1], actions)): #no final state, because it is a terminal state

                agent_cards, dealers_card = state
                agent_sum, useable_ace = self.calc_sum_useable_ace(agent_cards)

                if agent_sum < 12:
                    continue

                agent_sum_idx, dealers_card_idx, useable_ace_idx = self.map_to_indices(
                    agent_sum, dealers_card, useable_ace
                )
                actions_idx = self.actions_map[action]

                q_val = self.Q[
                    agent_sum_idx, dealers_card_idx, useable_ace_idx, actions_idx
                ]

                rho = self.is_ratio(states[i + 1 :], actions[i + 1 :])

                self.Q[
                    agent_sum_idx, dealers_card_idx, useable_ace_idx, actions_idx
                ] = q_val + self.alpha * (rho * reward - q_val)

                self.C[
                    agent_sum_idx, dealers_card_idx, useable_ace_idx, actions_idx
                ] += 1

        if track_history:
            self.Q_hist = pd.concat(Q_hist, axis=0)

        self.save()

    def plot_over_m(
        self, agent_sum, dealers_card, useable_ace, width=400, height=200, **kwargs
    ):
        """Creates a line plot of the action-value of hit and stick as episodes are processed. Note: we don't record
        action-value after every episodes, so not all action-values are shown."""

        if dealers_card == "A":
            dealers_card = f"'{dealers_card}'"

        df = self.Q_hist.query(
            f"(agent_sum == {agent_sum}) & (dealers_card == {dealers_card}) & (useable_ace == {useable_ace})"
        )[["Q", "action", "m"]]

        return (
            alt.Chart(df)
            .mark_line(**kwargs)
            .encode(x="m", y="Q", color="action")
            .properties(height=height, width=width)
        )

    def make_slice_df(self, Q_or_C_slice, name):
        df = pd.DataFrame(
            Q_or_C_slice, index=self.agent_sums, columns=self.dealers_cards
        )
        df.index.name = "agent_sum"
        df.columns.name = "dealers_card"
        return df.stack().to_frame(name)

    def get_df(self, which="Q"): #converting from numpy array to df

        assert which in ["Q", "C"]

        array = getattr(self, which)

        df = []
        for idx1 in [0, 1]: #usable aces (it only has two states)
            for idx2 in [0, 1]: #action (it only has two states)

                slc = array[:, :, idx1, idx2] # 4 slices in total. Ex: for every sums and delers card, get q value,
                                                        #where we had no ace and decided to stop hiting. Just like in his 4d graph.
                                                        #slc will be 2d array where rows = agent sums and columns = dealers cards
                slc_df = self.make_slice_df(slc, which) #becomes one column

                if which == "Q":
                    opt = array.argmax(-1)[:, :, idx1]
                    is_opt = opt == idx2

                    slc_df = pd.concat(
                        [slc_df, self.make_slice_df(is_opt.astype(int), "opt")], axis=1
                    )

                slc_df = slc_df.reset_index().assign(
                    useable_ace=self.ace_usability[idx1],
                    action=self.actions_possible[idx2],
                )
                df.append(slc_df)

        return pd.concat(df, axis=0)

    def test(self):
        agent_cards = self.sample_cards(2)  # Agent is delt 2 cards
        dealers_card = self.sample_cards(1)[0]  # Dealer is (effectively) delt 1 cards.
        print(agent_cards)
        print(dealers_card)
        states = [[agent_cards, dealers_card], [[1,2],3]]
        actions = []
        print(states)
        print(states[-1])
        print(states[-1][1])
        print([[1,2],2] + [3])
        print(list(range(0, 10000+1, 1000))[10])
        print([1,2,3,4,5][:-1])

        a = [1,2,3]
        b = ['a','b','c']
        #print(list(zip(a,b)))
        #for i, (state, action) in enumerate(zip(a, b)):
        #    print(action)
        print("-------------")
        df = pd.DataFrame({'A': range(1, 6),
                   'B': range(10, 0, -2),
                   'C C': range(10, 5, -1)})

        print(df[["A","B"]])

        a = np.array([[1,2,3],[4,5,6]])
        print(a[:,1])

        print("----")
        df = pd.DataFrame([[0, 1], [2, 3]],
                                    index=['cat', 'dog'],
                                    columns=['weight', 'height'])
        print(df)
        print(df.stack())
        print(type(df.stack()))
        print(df.stack().to_frame())
        print(type(df.stack().to_frame()))
        print(df.stack().to_frame().shape)

bj = BlackJack(M=500000, epsilon=0.1, alpha=1 / 500, seed=0)
bj.test()


import cv2
import numpy as np
import time

from adas_helper_functions import *
from ldw_helper_functions import *

countL = 0
countR = 0
c = 0
number = 0
counter = 0

ref_left = np.array([-0.0001, 0, 300])
ref_right = np.array([-0.0001, 0, 500])
left_fit = np.array([-0.0001, 0, 300])
right_fit = np.array([-0.0001, 0, 500])

currentL = np.zeros([3])
previousL = np.zeros([3])
currentR = np.zeros([3])
previousR = np.zeros([3])

DAY_BRIGHTNESS_TRESH = 30

def run_pipeline(frame):
    global ref_left, ref_right

    # Pre-processing

    """STEP#0 SCALE TO FIXED"""
    frame = scale_to_fixed(frame)
    # (h2, w2) = frame.shape[:2]

    brightness = getBrightness(frame)

    # ---------LDW PIPELINE

    """LDW-STEP#1 CROP ROAD ROI"""
    crop_road_roi_frame = crop_road_roi(frame, frame_output=False)

    """LDW-STEP#2 PERSPECTIVE TRANSFORM"""
    warped_image = apply_perspective_transform(frame, crop_road_roi_frame, toWarp=True)
    cv2.imshow("warp",warped_image)
    blur = cv2.GaussianBlur(warped_image, (5, 5), 0)
    """LDW-STEP#3 TIME CONDITION CHECK & BINARIZE"""
    if brightness > DAY_BRIGHTNESS_TRESH:
        # WHEN CLEAR BRIGHT DAY
        sharpened_image = sharpened(blur)
        binaryImage = compute_binary_image(warped_image, window_name="1st binary", output_show=False)
        # cv2.imshow("binary",binaryImage)
        whites = cv2.countNonZero(binaryImage)
        # print(whites)
        # WHITE COUNTS FOR WHEN IT RAINS
        if 0 < whites < 3000:
            binaryImage = edge_filter(warped_image, binaryImage)

        else:
            # morph top - remove big region
            # morph open - remove small region ie. erosion
            kernel_1 = cv2.getStructuringElement(cv2.MORPH_RECT, (9, 9))
            environment = cv2.morphologyEx(binaryImage, cv2.MORPH_TOPHAT, kernel_1)
            kernel_2 = cv2.getStructuringElement(cv2.MORPH_RECT, (3, 3))
            binaryImage = cv2.morphologyEx(environment, cv2.MORPH_OPEN, kernel_2)

    else:
        # WHEN IT IS NOT CLEAR BRIGHT DAY (NIGHT || UNDER BRIDGE)
        # denoise = cv2.fastNlMeansDenoisingColored(warped_image)
        sharpened_image = sharpened(warped_image)
        # sharpened_image = cv2.bilateralFilter(sharpened_image, 9, 18, 9 / 2)
        # cv2.imshow("bila",sharpened_image)
        binaryImage1 = compute_binary_image(sharpened_image, window_name="1st binary", output_show=False)
        binaryImage = edge_filter(warped_image, binaryImage1)

        # whites = cv2.countNonZero(binaryImage)

    """LDW-STEP#4 DEFINE LANE-LINES"""
    leftx, lefty, rightx, righty, left_lane_inds, right_lane_inds = extract_lanes_pixels(binaryImage, plot_show=True)
    left_fit, right_fit = poly_fit(leftx, lefty, rightx, righty, left_lane_inds, right_lane_inds,
                                   binaryImage, output_show=False)

    """LDW-STEP#5 SANITY CHECK"""
    sanity_stat = sanity_check(leftx, rightx, left_fit, right_fit, 0, .25)

    """LDW-STEP#6 DEAL WITH RELIABLE & UNRELIABLE FIT"""
    if sanity_stat:
        # Save as last known reliable fit
        ref_left, ref_right = left_fit, right_fit
        left_fit_up, right_fit_up = left_fit, right_fit
    else:
        if len(leftx) == 0:
            right = right_fit[0] * 540 ** 2 + right_fit[1] * 540 + right_fit[2]
            left_fit_up = np.array([-0.0001, 0, right - 200])
            right_fit_up = right_fit

        elif len(rightx) == 0:
            left = left_fit[0] * 540 ** 2 + left_fit[1] * 540 + left_fit[2]
            right_fit_up = np.array([-0.0001, 0, left + 200])
            left_fit_up = left_fit

        else:
            left_fit_up, right_fit_up = ref_left, ref_right

    """LDW-STEP#7 TRIGGER WARNING"""
    color, center = warning(left_fit_up, right_fit_up)

    if color == 1:
        color = (0, 0, 255)
    elif color == 2:
        color = (0, 255, 255)
    else:
        color = (255, 215, 0)

    warped_color_img = drawLine(frame, binaryImage, left_fit_up, right_fit_up, color, int(center))

    dewarped_image = apply_perspective_transform(frame, warped_color_img,  toWarp=False)

    frame = cv2.addWeighted(dewarped_image, 0.8, frame, 1.0, 0)
    cv2.line(frame, (int(640 / 2), 360), (int(640 / 2), 0), (0, 0, 255), 1)

    return frame

# source_frame = cv2.VideoCapture("C:/Users/default.DESKTOP-O651TTO/Documents/LDWS/SiangHW.mp4") #Day time video
# source_frame = cv2.VideoCapture("C:/Users/default.DESKTOP-O651TTO/Documents/Video ADAS/WIN_20180803_205421.MP4") # Night time video
# source_frame = cv2.VideoCapture("C:/Users/default.DESKTOP-O651TTO/Documents/Video ADAS/WIN_20180821_114234.MP4")
source_frame = cv2.VideoCapture("C:\\Users\\Sayang Nurul\\Lane detection2\\640 x 360\\Straight Lane.mp4")#Siang
 #  source_frame = cv2.VideoCapture("C:\\Users\\lenovo\\Documents\\Video ADAS\\WIN_20180803_205421.MP4") #mlm

_, test_frame = source_frame.read()
(h, w) = test_frame.shape[:2]
createtrackbar(h, w)
start = time.time()
count=0
while True:
    _, frame = source_frame.read()

    final_merge = run_pipeline(frame)

    cv2.imshow("FINAL-OUTPUT", final_merge)
    count=count+1;
    key = cv2.waitKey(1) & 0xFF
    if key == ord("q"):
        break

end = time.time()
duration = end-start
print("time taken: ",duration, count)
source_frame.release()
cv2.destroyAllWindows()


import torch
from PIL import Image
from mmocr.utils.ocr import MMOCR
from torchvision import transforms

to_tensor = transforms.ToTensor()

if __name__ == "__main__":

    # 导入模型到内存
    ocr = MMOCR(det='DB_r50', recog=None)
    img = Image.open(r'F:\OCR-TASK\OCR_Dataset\Other_back\0.png')
    results,feamap = ocr.readtext(to_tensor(img),
                                  output='./', export='./',
                                  feaName=["neck.lateral_convs.0",
                                           "neck.lateral_convs.1",
                                           "neck.lateral_convs.2",
                                           "neck.lateral_convs.3",
                                           "bbox_head.binarize.7"])

    print(feamap[0].fea.shape)
    print(feamap[1].fea.shape)
    print(feamap[2].fea.shape)
    print(feamap[3].fea.shape)
    print(torch.max(feamap[4].fea))


from zope.interface import Attribute, Interface
from zope.schema import Dict, Text, TextLine, URI

from pyams_chat import _

CHAT_PING_ROUTE = 'chat.ping'
CHAT_WORKER_ROUTE = 'chat.worker'

REST_CONTEXT_ROUTE = 'chat.rest.context'
REST_NOTIFICATIONS_ROUTE = 'chat.rest.notifications'

class IChatMessage(Interface):
    """Chat message interface"""

    host = TextLine(title=_("Message host name"))

    channel = TextLine(title=_("Message channel"))

    action = TextLine(title=_("Message action"))

    category = TextLine(title=_("Message category"))

    status = TextLine(title=_("Message status"))

    title = TextLine(title=_("Message title"))

    message = Text(title=_("Message content"))

    image = URI(title=_("Message image URL"),
                required=False)

    source = Dict(title=_("Message source"),
                  description=_("Attributes of the principal which emitted the notification"),
                  key_type=TextLine(),
                  value_type=TextLine())

    target = Dict(title=_("Message target"),
                  description=_("Message targets can be principals, roles..."),
                  key_type=TextLine())

    url = TextLine(title=_("Notification URL"),
                   description=_("URL target of this message notification"),
                   required=False)

    timestamp = TextLine(title=_("Message timestamp in ISO-8601 format"))

    user_data = Dict(title=_("Internal user data"),
                     description=_("User data used for internal usage; these data are not "
                                   "sent in notifications contents"))

    def send(self):
        """Send notification to recipients"""

class IChatMessageExtension(Interface):
    """Chat message extension interface

    These extensions, defined as adapters, can be used to add custom
    information to an existing message.

    This interface doesn't provide any method, as adapters should
    directly extend the original message.
    """

    weight = Attribute("Optional ordering weight")

class IChatMessageHandler(Interface):
    """Chat message handler interface

    This interface is used to define adapters which can be used to
    define a message target based on it's category.
    """

    weight = Attribute("Optional ordering weight")

    def get_target(self):
        """Get message target"""

class IRedisClient(Interface):
    """Redis client interface for chat"""


import asyncio
import contextlib
import datetime
import hashlib
import mmap
import os
import os.path
import random
import subprocess
import sys
import tempfile
import time
from typing import IO, Callable
from zoneinfo import ZoneInfo

import humanize
import structlog
import tzlocal

from .ext_deps import CP
from .fallocate import punch_hole

log = structlog.stdlib.get_logger(subsystem="utils")

log_data: str  # for pytest

Bytes = 1
kiB = Bytes * 1024
MiB = kiB * 1024
GiB = MiB * 1024
TiB = GiB * 1024

# 64 kiB blocks are a good compromise between sparsiness and fragmentation.
PUNCH_SIZE = 4 * MiB
CHUNK_SIZE = 4 * MiB

END = object()

def report_status(f):
    def wrapped(*args, **kw):
        generator = iter(f(*args, **kw))
        steps = next(generator)
        step = 0
        start = time.time()
        while True:
            status = next(generator)
            if status is END:
                break
            step += 1
            now = time.time()
            time_elapsed = now - start
            per_chunk = time_elapsed / step
            remaining = steps - step
            time_remaining = remaining * per_chunk
            eta = datetime.datetime.now() + datetime.timedelta(
                seconds=int(time_remaining)
            )
            if step == 5 or not step % 100:  # pragma: nocover
                log.info(
                    "progress-report",
                    step=step,
                    steps=steps,
                    percent=round(step / steps * 100, 2),
                    elapsed=round(time_elapsed),
                    remaining=humanize.naturaldelta(time_remaining),
                    eta=str(eta),
                )

        result = next(generator)
        return result

    return wrapped

class SafeFile(object):
    """A context manager for handling files in our
    scenarios more safely:

    - manage use of temporary files and os.rename for committing
    - ensure files are flushed to disk when closing
    - ensure files are write-protected (and maybe temporarily not)

    Use this as a context manager and then call the various open_*
    methods or use_write_protection() to enable the safety belts.

    use_write_protection() has to be called before opening.
    """

    protected_mode = 0o440

    def __init__(self, filename, encoding=None, sync=True):
        self.filename = filename
        self.encoding = encoding
        self.sync = sync
        self.f = None

    def __enter__(self):
        assert self.f is None
        self.rename = False
        self.write_protected = False
        self.commit_callbacks = []
        return self

    def __exit__(self, exc_type=None, exc_value=None, exc_tb=None):
        # Error handling:
        # - if we're based on temporary files, we simply unlink
        #   and haven't changed the original
        # - if we're working with the original, we at least have
        #   synced by now and will restore write-protection
        if self.f is None:
            return
        if self.sync:
            self.f.flush()
            os.fsync(self.f)
        self.f.close()

        if self.use_write_protection:
            os.chmod(self.f.name, self.protected_mode)
            if os.path.exists(self.filename):
                os.chmod(self.filename, self.protected_mode)

        if self.rename:
            if exc_type is None:
                os.rename(self.f.name, self.filename)
            else:
                os.unlink(self.f.name)

        self.f = None

    # Activate the different safety features

    def open_new(self, mode):
        """Open this as a new (temporary) file that will be renamed on close."""
        assert not self.f
        self.f = tempfile.NamedTemporaryFile(
            mode, dir=os.path.dirname(self.filename), delete=False
        )
        self.rename = True

    def open_copy(self, mode):
        """Open an existing file, make a copy first, rename on close."""
        assert not self.f

        if os.path.exists(self.filename):
            self.open_new("wb")
            cp_reflink(self.filename, self.f.name)
            self.f.close()

        self.f = open(self.f.name, mode)

    def open_inplace(self, mode):
        """Open as an existing file, in-place."""
        assert not self.f
        if self.use_write_protection and os.path.exists(self.filename):
            # This is kinda dangerous, but this is a tool that only protects
            # you so far and doesn't try to get in your way if you really need
            # to do this.
            os.chmod(self.filename, 0o640)
        self.f = open(self.filename, mode)

    def use_write_protection(self):
        """Enable write-protection handling.

        Ensures you can do your work (non-write-protected) and ensures to
        (re-)enable write protection on close.

        """
        assert not self.f
        self.write_protected = True

    # File API shim

    @property
    def name(self):
        return self.f.name

    def read(self, *args, **kw):
        data = self.f.read(*args, **kw)
        if self.encoding:
            data = data.decode(self.encoding)
        return data

    def write(self, data):
        if self.encoding:
            data = data.encode(self.encoding)
        self.f.write(data)

    def seek(self, offset, whence=0):
        return self.f.seek(offset, whence)

    def tell(self):
        return self.f.tell()

    def truncate(self, size=None):
        return self.f.truncate(size)

    def fileno(self):
        return self.f.fileno()

if hasattr(os, "posix_fadvise"):
    posix_fadvise = os.posix_fadvise
else:  # pragma: no cover
    os.POSIX_FADV_RANDOM = None  # type: ignore
    os.POSIX_FADV_SEQUENTIAL = None  # type: ignore
    os.POSIX_FADV_WILLNEED = None  # type: ignore
    os.POSIX_FADV_DONTNEED = None  # type: ignore

    def posix_fadvise(*args, **kw):
        log.debug("posix_fadivse-unavailable")
        return

if sys.platform == "darwin":  # pragma: no cover

    @contextlib.contextmanager
    def zeroes(size):
        yield "\00" * size

else:

    @contextlib.contextmanager
    def zeroes(size):
        with open("/dev/zero", "rb", buffering=0) as f:
            z = mmap.mmap(f.fileno(), size, access=mmap.ACCESS_READ)
            yield z[0:size]
            z.close()

@report_status
def copy_overwrite(source, target):
    """Efficiently overwrites `target` with a copy of `source`.

    Identical regions won't be touched so this is COW-friendly. Assumes
    that `target` is a shallow copy of a previous version of `source`.

    Assumes that `target` exists and is open in read-write mode.
    """
    punch_size = PUNCH_SIZE
    source.seek(0, 2)
    size = source.tell()
    if size > 500 * GiB:
        punch_size *= 10
    source.seek(0)
    yield size / punch_size
    try:
        posix_fadvise(source.fileno(), 0, 0, os.POSIX_FADV_SEQUENTIAL)
        posix_fadvise(target.fileno(), 0, 0, os.POSIX_FADV_SEQUENTIAL)
    except Exception:
        pass
    with zeroes(punch_size) as z:
        while True:
            startpos = source.tell()
            chunk = source.read(punch_size)
            if not chunk:
                break
            target.seek(startpos)
            compare = target.read(len(chunk))
            if not compare or chunk != compare:
                if chunk == z:
                    target.flush()
                    punch_hole(target, startpos, len(chunk))
                else:
                    target.seek(startpos)
                    target.write(chunk)
            yield

    size = source.tell()
    target.flush()
    try:
        target.truncate(size)
    except OSError:
        pass  # truncate may not be supported, i.e. on special files
    try:
        os.fsync(target)
    except OSError:
        pass  # fsync may not be supported, i.e. on special files
    yield END
    yield size

@report_status
def copy(source, target):
    """Efficiently overwrites `target` with a copy of `source`.

    Identical regions will be touched - so this is not CoW-friendly.

    Assumes that `target` exists and is open in read-write mode.
    """

    chunk_size = 4 * 1024 * 1024
    source.seek(0, 2)
    yield source.tell() / chunk_size
    source.seek(0)

    try:
        posix_fadvise(source.fileno(), 0, 0, os.POSIX_FADV_SEQUENTIAL)
        posix_fadvise(target.fileno(), 0, 0, os.POSIX_FADV_SEQUENTIAL)
    except Exception:
        pass
    while True:
        chunk = source.read(chunk_size)
        if not chunk:
            break
        target.write(chunk)
        yield

    size = source.tell()
    target.flush()
    try:
        target.truncate(size)
    except OSError:
        pass  # truncate may not be supported, i.e. on special files
    try:
        os.fsync(target)
    except OSError:
        pass  # fsync may not be supported, i.e. on special files
    yield END
    yield size

def cp_reflink(source, target):
    """Makes as COW copy of `source` if COW is supported."""
    # We can't tell if reflink is really supported. It depends on the
    # filesystem.
    try:
        subprocess.check_call(
            [CP, "--reflink=always", source, target], stderr=subprocess.PIPE
        )
    except subprocess.CalledProcessError:
        log.warning("cp_reflink-no-cow", source=source, target=target)
        if os.path.exists(target):
            os.unlink(target)
        subprocess.check_call([CP, source, target])

def files_are_equal(
    a: IO,
    b: IO,
    report: Callable[[bytes, bytes, int], None] = lambda a, b, c: None,
) -> bool:
    try:
        posix_fadvise(a.fileno(), 0, 0, os.POSIX_FADV_SEQUENTIAL)  # type: ignore
        posix_fadvise(b.fileno(), 0, 0, os.POSIX_FADV_SEQUENTIAL)  # type: ignore
    except Exception:
        pass
    position = 0
    errors = 0
    while True:
        chunk_a = a.read(CHUNK_SIZE)
        chunk_b = b.read(CHUNK_SIZE)
        if chunk_a != chunk_b:
            log.error(
                "files-not-equal",
                hash_a=hashlib.md5(chunk_a).hexdigest(),
                hash_b=hashlib.md5(chunk_b).hexdigest(),
                pos=position,
            )
            report(chunk_a, chunk_b, position)
            errors += 1
        if not chunk_a:
            break
        position += CHUNK_SIZE
    return not errors

def files_are_roughly_equal(
    a: IO,
    b: IO,
    samplesize=0.01,
    blocksize=CHUNK_SIZE,
    timeout=5 * 60,
    report: Callable[[bytes, bytes, int], None] = lambda a, b, c: None,
) -> bool:
    a.seek(0, os.SEEK_END)
    size = a.tell()
    blocks = size // blocksize
    blocklist = range(0, max(blocks, 1))
    sample = random.sample(blocklist, max(int(samplesize * blocks), 1))

    # turn off readahead
    try:
        for fdesc in a.fileno(), b.fileno():
            posix_fadvise(fdesc, 0, 0, os.POSIX_FADV_RANDOM)  # type: ignore
    except Exception:
        pass

    # We limit this to a 5 minute operation to avoid clogging the storage
    # infinitely.
    started = now()
    max_duration = datetime.timedelta(seconds=timeout)

    for block in sample:
        log.debug("files-roughly-equal-verifying", offset=block * blocksize)
        duration = now() - started
        if duration > max_duration:
            log.info("files-roughly-equal-stopped", duration=duration)
            return True

        a.seek(block * blocksize)
        b.seek(block * blocksize)
        chunk_a = a.read(blocksize)
        chunk_b = b.read(blocksize)
        if chunk_a != chunk_b:
            log.error(
                "files-not-roughly-equal",
                hash_a=hashlib.md5(chunk_a).hexdigest(),
                hash_b=hashlib.md5(chunk_b).hexdigest(),
                pos=block * blocksize,
            )
            report(chunk_a, chunk_b, block * blocksize)
            return False
    return True

def now():
    """A monkey-patchable version of 'datetime.datetime.now()' to
    support unit testing.

    Also, ensure that we'll always use timezone-aware objects.
    """
    return datetime.datetime.now(ZoneInfo("UTC"))

def min_date():
    return datetime.datetime.min.replace(tzinfo=ZoneInfo("UTC"))

def has_recent_changes(entry, reference_time):
    # This is not efficient on a first look as we may stat things twice, but it
    # makes the recursion easier to read and the VFS will be caching this
    # anyway.
    # However, I want to perform a breadth-first analysis as the theory is that
    # higher levels will propagate changed mtimes do to new/deleted files
    # instead of just modified files in our case and looking at stats when
    # traversing a directory level is faster than going depth first.
    st = entry.stat(follow_symlinks=False)
    if st.st_mtime >= reference_time:
        return True
    if not entry.is_dir(follow_symlinks=False):
        return False
    candidates = list(os.scandir(entry.path))
    # First pass: stat all direct entries
    for candidate in candidates:
        if candidate.stat(follow_symlinks=False).st_mtime >= reference_time:
            return True
    # Second pass: start traversing
    for candidate in os.scandir(entry.path):
        if has_recent_changes(candidate, reference_time):
            return True
    return False

async def time_or_event(deadline, event):
    remaining_time = (deadline - now()).total_seconds()
    return await next(
        asyncio.as_completed([asyncio.sleep(remaining_time), event.wait()])
    )

def format_datetime_local(dt):
    tz = tzlocal.get_localzone()
    if dt is None:
        return "-", tz
    return (
        dt.astimezone(tz).replace(tzinfo=None).strftime("%Y-%m-%d %H:%M:%S"),
        tz,
    )