import numpy as np # linear algebra
import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)
import seaborn as sns
from functools import reduce
from matplotlib import pyplot
import os
import gc
from sklearn.ensemble import ExtraTreesClassifier
PATH = "../input"

# Preview of Categorical variables in a dataset

def data_preview_str(df, name):
    tot_feats = 0
    if name=='application_train':
        for col in [i for i in df.columns if (df[i].dtypes=='O') or (len(df[i].unique())<=10)]:
            tot_feats = tot_feats + len(df[col].unique())-1
            print(" %s "% col, '# unique: ',len(df[col].unique()))
            xdf = df.groupby(col).agg({'SK_ID_CURR':'count','TARGET':'sum'})
            xdf.columns = ['#Records','Response_Rate']
            xdf['Response_Rate'] = xdf['Response_Rate']/sum(xdf['Response_Rate'])
            xdf['%Records'] = xdf['#Records']*100/sum(xdf['#Records'])
            print(xdf.loc[:,['#Records','%Records','Response_Rate']])
            del xdf
            print("-"*50)
    else:
        for col in [i for i in df.columns if (df[i].dtypes=='O') or (len(df[i].unique())<=15)]:
            tot_feats = tot_feats + len(df[col].unique())-1
            print(" %s "% col, '# unique: ',len(df[col].unique()))
            xdf = pd.DataFrame(df[col].value_counts())
            xdf['%Records'] = xdf[col]*100/sum(xdf[col])
            xdf.columns = ['#Records','%Records']
            print(xdf)
            del xdf
            print("-"*50)
    ll=[i for i in df.columns if (df[i].dtypes!='O') and (len(df[i].unique())>15)]
    tot_feats= tot_feats + len(ll)
    print("Creating dummies will lead to a total of %d features"% tot_feats)
    return 1

# MIssing data info
def missing_data(data, size):
    total = data.isnull().sum().sort_values(ascending = False)
    percent = (data.isnull().sum()/data.isnull().count()*100).sort_values(ascending = False)
    mdata = pd.concat([total, percent], axis=1, keys=['Total', 'Percent'])
    mdata = mdata.loc[mdata['Percent']>0,:].reset_index()
    if mdata.shape[0]==0:
        print("No missing data!")
        return 1
    else:
        sns.set_style("whitegrid")
        fig, ax =pyplot.subplots(figsize=size)
        ax = sns.barplot(x="Percent", y="index", data=mdata)
        return ax

# Create flags for Missing info
def missing_info_flag(df):
    mia_col_list = [i for i in df.columns if df[i].isnull().values.any().any()]
    counter = 0
    for col in mia_col_list:
        df[col+'_mia_'] = np.where(df[col].isnull(),1,0)
        counter +=1
    print("Created %d missing flags"% counter)

# Create dummy variables
def create_dummy_vars(df):
    dummy_varlist = [i for i in df.columns if df[i].dtypes=='O']
    dummy_df = pd.get_dummies(df, columns = dummy_varlist)
    return dummy_df
#     return pd.merge(df[[i for i in df.columns if df[i].dtypes!='O']],dummy_df, how='left', left_index=True, right_index = True)

# Pairplot for distribution of contnuous variables
def xdistributions(df, cols, kde):
    ncols = [i for i in cols if df[i].dtypes!='O']
    df.loc[:,'TARGET_char'] = np.where(df['TARGET']==0,'Clean','Default')
    if kde:
        ax = sns.pairplot(df.loc[:,ncols+['TARGET_char']], hue='TARGET_char', diag_kind='kde', diag_kws=dict(shade=True), plot_kws={'line_kws':{'color':'red'}, 'scatter_kws': {'alpha': 0.1}})
    else:
        ax = sns.pairplot(df.loc[:,ncols+['TARGET_char']], hue='TARGET_char', plot_kws={'line_kws':{'color':'red'}, 'scatter_kws': {'alpha': 0.1}})
    df.drop('TARGET_char', axis=1, inplace=True)
    return ax

def prep_data(df):
    print("Before creating dummy variables #Columns: %d"% df.shape[1])
    p_df = create_dummy_vars(df)
    print("After creating dummy variables #Columns: %d"% p_df.shape[1])

    missing_info_flag(p_df)

    p_df = p_df.fillna(p_df.median())
    return p_df

# THis function calculates mean min max median for continuous columns and mena for dummy column sat SK_ID_CURR level
def agg_at_skid(df):
    numerics = ['int16', 'int32', 'int64', 'float16', 'float32', 'float64']
    flags = [ i for i in df.columns if (df[i].dtypes!='O') and (len(df[i].unique())<=2)]
    non_flags = [ i for i in df.columns if (df[i].dtypes!='O') and (len(df[i].unique())>2)]
    print("# unique SK_ID_CURR : ",len(df['SK_ID_CURR'].unique()))
    df_at_skid = pd.merge(df.loc[:,non_flags].groupby('SK_ID_CURR').agg(['min','max','mean','median']),
                              df.loc[:,flags+['SK_ID_CURR']].groupby('SK_ID_CURR').agg(['mean']),
                              how='inner', left_index=True, right_index=True)
    print("After merging shape : ",df_at_skid.shape)
    df_at_skid = pd.DataFrame(df_at_skid)
    df_at_skid.columns = ['_'.join(col) for col in df_at_skid.columns]
    df_at_skid.reset_index(inplace=True)
    df_at_skid.sort_values(by='SK_ID_CURR', ascending = [1])
    df_at_skid.reset_index(inplace=True, drop=True)
    return df_at_skid

def lgbm(data, test, y, **kwargs):
    from sklearn.metrics import roc_auc_score, roc_curve
    from sklearn.model_selection import KFold
    from lightgbm import LGBMClassifier
    import gc
    # Get features
    excluded_feats = ['SK_ID_CURR','TARGET']
    features = [f_ for f_ in data.columns if f_ not in excluded_feats]

    # Run a 3 fold
    folds = KFold(n_splits=3, shuffle=True, random_state=123451)
    oof_preds = np.zeros(data.shape[0])
    sub_preds = np.zeros(test.shape[0])
    feature_importance_df = pd.DataFrame()
    print(kwargs)
    if kwargs !={}:
        for key, value in kwargs.items():
            print("The value of {} is {}".format(key, value))
    else:
        n_estimators=1500
        learning_rate = 0.01
        num_leaves = 50
        max_depth = 6
        colsample_bytree = .8
        subsample =.9
        max_depth =6
        reg_alpha =.1
        reg_lambda =.1
        min_split_gain =.01
        min_child_weight =2
        silent =True
        verbose =-1

    for n_fold, (trn_idx, val_idx) in enumerate(folds.split(data)):
        trn_x, trn_y = data[features].iloc[trn_idx], y.iloc[trn_idx]
        val_x, val_y = data[features].iloc[val_idx], y.iloc[val_idx]

        clf = LGBMClassifier(
            n_estimators=n_estimators,
            learning_rate=learning_rate,
            num_leaves=num_leaves,
            colsample_bytree=colsample_bytree,
            subsample=subsample,
            max_depth=max_depth,
            reg_alpha=reg_alpha,
            reg_lambda=reg_lambda,
            min_split_gain=min_split_gain,
            min_child_weight=min_child_weight,
            silent=silent,
            verbose=verbose,
            early_stopping_rounds=10
        )

        clf.fit(trn_x, trn_y,
                eval_set= [(trn_x, trn_y), (val_x, val_y)],
                eval_metric='auc', verbose=100, early_stopping_rounds=150
               )

        oof_preds[val_idx] = clf.predict_proba(val_x, num_iteration=clf.best_iteration_)[:, 1]
        sub_preds += clf.predict_proba(test[features], num_iteration=clf.best_iteration_)[:, 1] / folds.n_splits

        print('Fold %2d AUC : %.6f' % (n_fold + 1, roc_auc_score(val_y, oof_preds[val_idx])))

        fold_importance_df = pd.DataFrame()
        fold_importance_df["feature"] = features
        fold_importance_df["importance"] = clf.feature_importances_
        fold_importance_df["fold"] = n_fold + 1
        feature_importance_df = pd.concat([feature_importance_df, fold_importance_df], axis=0)

        del clf, trn_x, trn_y, val_x, val_y
        gc.collect()

    print('Full AUC score %.6f' % roc_auc_score(y, oof_preds))
    return sub_preds, feature_importance_df
gc.collect()
application_train = pd.read_csv(PATH+"/application_train.csv")
print('Size of application_train '+'{:,}'.format(application_train.shape[0])+' rows and '+'{:,}'.format(application_train.shape[1])+' columns')
missing_data(application_train, size=(15,15))
app_tr_cont_cols = [i for i in application_train.columns if (application_train[i].dtypes!='O') and (len(application_train[i].unique())>10)]
application_train.loc[:,app_tr_cont_cols].describe()
data_preview_str(application_train,'application_train')
missing_info_flag(application_train)
application_train = prep_data(application_train)

# bureau
bureau = pd.read_csv(PATH+"/bureau.csv")
cols_ = [i for i in bureau.columns if (bureau[i].dtypes!='O') and (len(bureau[i].unique())>10)]
bureau.loc[:,cols_].describe()
missing_data(bureau,(5,5))
data_preview_str(bureau,'bureau_balance')
bureau = prep_data(bureau)
bureau_at_skid = agg_at_skid(bureau)
del bureau
import gc
gc.collect()
POS_CASH_balance = pd.read_csv(PATH+"/POS_CASH_balance.csv")
print('POS_CASH_balance '+'{:,}'.format(POS_CASH_balance.shape[0])+' rows and '+'{:,}'.format(POS_CASH_balance.shape[1])+' columns')
cols_ = [i for i in POS_CASH_balance.columns if (POS_CASH_balance[i].dtypes!='O') and (len(POS_CASH_balance[i].unique())>10)]
POS_CASH_balance.loc[:,cols_].describe()
missing_data(POS_CASH_balance,(5,5))
data_preview_str(POS_CASH_balance,'POS_CASH_balance')
POS_CASH_balance = prep_data(POS_CASH_balance)
POS_CASH_balance_at_skid = agg_at_skid(POS_CASH_balance)
del POS_CASH_balance
gc.collect()
credit_card_balance = pd.read_csv(PATH+"/credit_card_balance.csv")
print('credit_card_balance '+'{:,}'.format(credit_card_balance.shape[0])+' rows and '+'{:,}'.format(credit_card_balance.shape[1])+' columns')
cols_ = [i for i in credit_card_balance.columns if (credit_card_balance[i].dtypes!='O') and (len(credit_card_balance[i].unique())>10)]
credit_card_balance.loc[:,cols_].describe()
missing_data(credit_card_balance,(5,5))
data_preview_str(credit_card_balance,'credit_card_balance')
credit_card_balance = prep_data(credit_card_balance)
credit_card_balance_at_skid = agg_at_skid(credit_card_balance)
del credit_card_balance
gc.collect()
bureau_balance = pd.read_csv(PATH+"/bureau_balance.csv")
print('bureau_balance '+'{:,}'.format(bureau_balance.shape[0])+' rows and '+'{:,}'.format(bureau_balance.shape[1])+' columns')
cols_ = [i for i in bureau_balance.columns if (bureau_balance[i].dtypes!='O') and (len(bureau_balance[i].unique())>10)]
bureau_balance.loc[:,cols_].describe()
missing_data(bureau_balance,(5,5))
data_preview_str(bureau_balance,'bureau_balance')
gc.collect()
# bureau_balance = prep_data(bureau_balance)
# bureau_balance_at_skid = agg_at_skid(bureau_balance)
del bureau_balance
gc.collect()
# previous_application
previous_application = pd.read_csv(PATH+"/previous_application.csv")
print('previous_application '+'{:,}'.format(previous_application.shape[0])+' rows and '+'{:,}'.format(previous_application.shape[1])+' columns')
cols_ = [i for i in previous_application.columns if (previous_application[i].dtypes!='O') and (len(previous_application[i].unique())>10)]
previous_application.loc[:,cols_].describe()
missing_data(previous_application,(5,5))
data_preview_str(previous_application,'previous_application')
# previous_application = prep_data(previous_application)
# previous_application_at_skid = agg_at_skid(previous_application)
del previous_application
gc.collect()
# installments_payments
print('installments_payments '+'{:,}'.format(installments_payments.shape[0])+' rows and '+'{:,}'.format(installments_payments.shape[1])+' columns')
cols_ = [i for i in installments_payments.columns if (installments_payments[i].dtypes!='O') and (len(installments_payments[i].unique())>10)]
missing_data(installments_payments,(10,10))
data_preview_str(installments_payments,'installments_payments')
del installments_payments
gc.collect()
df_list = [application_train, bureau_at_skid, POS_CASH_balance_at_skid, installments_payments_at_skid, credit_card_balance_at_skid] #, bureau_balance_at_skid previous_application_at_skid
print("Before merging: ",application_train.shape)
X_train = reduce(lambda left,right: pd.merge(left,right,how='left',on='SK_ID_CURR'), df_list)
print("After merging: ",X_train.shape)
missing_data(X_train,(50,50))
del application_train
gc.collect()
application_test = pd.read_csv(PATH+"/application_test.csv")
missing_info_flag(application_test)
application_test = prep_data(application_test)
X_train.columns.values
X_train.drop([ 'SK_ID_PREV_min', 'SK_ID_PREV_max', 'SK_ID_PREV_mean','SK_ID_PREV_median'], axis=1, inplace=True)
df_list = [application_test, bureau_at_skid, POS_CASH_balance_at_skid, installments_payments_at_skid, credit_card_balance_at_skid]  #, bureau_balance_at_skid previous_application_at_skid
print("Before merging: ",application_test.shape)
X_test = reduce(lambda left,right: pd.merge(left,right,how='left',on='SK_ID_CURR'), df_list)
print("After merging: ",X_test.shape)
missing_data(X_test,(50,150))
X_test.drop(['SK_ID_PREV_min', 'SK_ID_PREV_max', 'SK_ID_PREV_mean','SK_ID_PREV_median'], axis=1, inplace=True)
del application_test, bureau_at_skid, POS_CASH_balance_at_skid, credit_card_balance_at_skid, installments_payments_at_skid #previous_application_at_skid
gc.collect()
# Dropping some ID aggregated variables
# dropvars = ['SK_ID_PREV_min_y', 'SK_ID_PREV_max_y', 'SK_ID_PREV_mean_y', 'SK_ID_PREV_median_y','SK_ID_BUREAU_min', 'SK_ID_BUREAU_max', 'SK_ID_BUREAU_mean', 'SK_ID_BUREAU_median']
# X_train.drop(dropvars, axis=1, inplace=True)
# X_test.drop(dropvars, axis=1, inplace=True)
X_train = X_train.fillna(X_train.median())
X_test = X_test.fillna(X_test.median())
gc.collect()
# Remove variables where the value is same for more than 99% pf training sets
def variance_threshold_selector(data, threshold):
    from sklearn.feature_selection import VarianceThreshold
    print("# features before Variance Threshold: ", data.shape[1])
    selector = VarianceThreshold(threshold)
    selector.fit(data)
    return data[data.columns[selector.get_support(indices=True)]]

X_test_sel = variance_threshold_selector(data=X_test, threshold=0.99)
print("# features after Variance Threshold: ", X_test_sel.shape[1])
X_train_sel = variance_threshold_selector(data=X_train, threshold=0.99)
print("# features after Variance Threshold: ", X_train_sel.shape[1])
y = X_train['TARGET']
gc.collect()
del X_test, X_train
gc.collect()
preds, feature_importance_df = lgbm(data=X_train_sel.loc[:,[i for i in X_train_sel.columns if i in X_test_sel.columns]], test=X_test_sel, y=y)
X_train_sel.drop([i for i in X_train_sel.columns if i not in X_test_sel.columns], axis=1, inplace=True)
gc.collect()
X_test_sel['TARGET'] = preds

X_test_sel[['SK_ID_CURR', 'TARGET']].to_csv('take0_submission.csv', index=False, float_format='%.8f')
# Plot feature importances
import matplotlib.pyplot as plt
cols = feature_importance_df[["feature", "importance"]].groupby("feature").mean().sort_values(
    by="importance", ascending=False)[:50].index

best_features = feature_importance_df.loc[feature_importance_df.feature.isin(cols)]

plt.figure(figsize=(8,10))
sns.barplot(x="importance", y="feature", data=best_features.sort_values(by="importance", ascending=False))
plt.title('LightGBM Features (avg over folds)')
plt.tight_layout()
plt.savefig('lgbm_importances.png')
# del X_test
# gc.collect()


#!/usr/bin/env python
from io import open
from setuptools import setup, find_packages

with open("README.md", "r") as f:
    long_description = f.read()

setup(
    name="wheeltools",
    use_scm_version={"write_to": "src/wheeltools/_version.py"},
    description="General tools for working with wheels",
    long_description=long_description,
    long_description_content_type="text/markdown",
    author="Matthew Brett",
    maintainer="Matthew Brett",
    author_email="matthew.brett@gmail.com",
    url="https://github.com/anthrotype/wheeltools",
    package_dir={"": "src"},
    packages=find_packages("src"),
    install_requires=["wheel"],
    setup_requires=["setuptools_scm"],
    extras_require={"testing": ["pytest >= 3.0.0, <4", "pytest-cov >= 2.5.1, <3"]},
    license="BSD license",
    classifiers=[
        "Intended Audience :: Developers",
        "Environment :: Console",
        "License :: OSI Approved :: BSD License",
        "Programming Language :: Python",
        "Operating System :: MacOS :: MacOS X",
        "Operating System :: POSIX :: Linux",
        "Development Status :: 4 - Beta",
        "Topic :: Software Development :: Libraries :: " "Python Modules",
        "Topic :: Software Development :: Build Tools",
    ],
)


import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import tensorflow as tf
import os
from sklearn.model_selection import train_test_split
tf.compat.v1.disable_eager_execution()
os.environ['TF_CPP_MIN_LOG_LEVEL'] = '2'

x_data=np.linspace(0,10,100)
noise=np.random.randn(100)

y_data=.3*x_data+5+noise
x_df=pd.DataFrame(data=x_data,columns=["x"])
y_df=pd.DataFrame(data=y_data,columns=["y"])

my_data=pd.concat([x_df,y_df],axis=1)
print(my_data)
#plt.scatter(my_data["x"],my_data["y"])
#plt.show()
x_d=my_data.iloc[:,0]
y_d=my_data.iloc[:,1]

#x_train,x_test,y_train,y_test=train_test_split(x_d,y_d,test_size=0.2,random_state=0)
xph=tf.compat.v1.placeholder(tf.float32,[len(x_d)])
yph=tf.compat.v1.placeholder(tf.float32,[len(x_d)])
feed={xph:x_d,yph:y_d}
w=tf.compat.v1.Variable(.3)
b=tf.compat.v1.Variable(5.0)

y_true=w*xph+b

error=tf.reduce_sum(tf.square(yph-y_true))

optimizer=tf.compat.v1.train.GradientDescentOptimizer(learning_rate=0.0001)
result=optimizer.minimize(error)
init=tf.compat.v1.global_variables_initializer()
sess=tf.compat.v1.Session()
sess.run(init)
sess.run(result,feed_dict=feed)
fw,fb=sess.run([w,b])
print(fw,fb)
y_final=x_d*fw+fb

print(y_final)
print(y_d)
plt.scatter(x_d,y_d)
plt.plot(x_d,y_final,color="red")
plt.show()


# =========================================================================== #
# Date: 06/12/19
# Author: Richard Lu
# Description:
#   train classification model
# Runtime:
# =========================================================================== #

import copy
import glob
import logging
import numpy as np
import os
import pickle
import random
import ujson
import warnings

from sklearn.ensemble import RandomForestClassifier
from sklearn.metrics import confusion_matrix, roc_curve, auc
from sklearn.model_selection import cross_val_score, cross_val_predict

# =========================================================================== #
# RUN DEPENDENCY SCRIPTS
# =========================================================================== #

warnings.filterwarnings("ignore")

os.chdir("../1_feature_engineering")
exec(open("extract_review_features.py").read())
exec(open("iterative_stratification.py").read())

handlabeled = ujson.load(open(
    "../../0_data/4_handlabeled_data/already_handlabeled_review_ids.json",
    "r"))
# handlabeled = ujson.load(open(
#     "../../0_data/4_handlabeled_data/already_handlabeled_review_ids_extended.json",
#     "r"))

handlabeled_set = set(handlabeled["labeled_ids"])

static_path = "/home/richard/Documents/work/insight/insight_project/0_data/2_processed_data"

review_filename = "review_count_by_business_id.csv"

review_count_data = pd.read_csv(os.path.join(static_path, review_filename)
    ).drop("Unnamed: 0", axis=1)

review_count_data["cum_sum"] = review_count_data["review_count"].cumsum()

max_review = review_count_data["review_count"].sum()

logging.basicConfig(filename="train_model.log", filemode="a",
level=logging.INFO,
format="%(asctime)s - %(message)s", datefmt="%d-%b-%y %H:%M:%S")

logging.info("Start script")

# =========================================================================== #
# TRAIN MODEL
# =========================================================================== #

feature_matrix = produce_feature_matrix(data2)
# all nas seem to be coming from syntax bigrams
# [('ADV_X', 1050), ('ADP_CONJ', 1050), ('NUM_X', 1050), ('PRON_CONJ', 1050), ('PRON_X', 1050), ('PRT_PRON', 1050), ('PRT_CONJ', 1050), ('PRT_X', 1050), ('DET_ADP', 1050), ('DET_PRON', 1050), ('DET_PRT', 1050), ('DET_CONJ', 1050), ('DET_X', 1050), ('CONJ_ADP', 1050), ('CONJ_PRON', 1050), ('CONJ_PRT', 1050), ('CONJ_DET', 1050), ('CONJ_X', 1050), ('X_ADP', 1050), ('X_PRON', 1050), ('X_PRT', 1050), ('X_DET', 1050), ('X_CONJ', 1050)]

# fill na with 0 just in case other reviews have those combinations

# na_columns = [x[0] for x in zip(feature_matrix.columns,
#                                 feature_matrix.isnull().sum().tolist())
#               if x[1] > 0]
feature_matrix = feature_matrix.fillna(0)

target = data[["food", "service", "price", "ambiance"]]

def train_pseudolabel_model(feature_matrix, target, target_string, iterations=0,
    step=3000, num_samples=20):
    """Train the classifier with iterations for pseudolabeling

    Args:
        feature_matrix - features
        target - target class data
        target_string - column name of intended target
        iterations - number of pseudolabeling iterations to run
        step - number of samples for each pseudolabel iteration
        num_samples - number of most confident samples to add if threshold
                     is adaptive

    Returns:
        final classifier

    """
    X_tmp, y_tmp, X_test, y_test = iterative_train_test_split(
        np.array(feature_matrix),
        np.array(target[[target_string]]),
        test_size=0.2,
        random_state=10191994)

    X_train, y_train, X_validation, y_validation = iterative_train_test_split(
        X_tmp,
        y_tmp,
        test_size=0.25, # 0.25 * 0.8 = 0.2
        random_state=10191994)

    with open("../../0_data/5_train_validation_test/X_train_{}_1150.pickle".format(
        target_string), "wb") as f:
        pickle.dump(X_train, f)
    with open("../../0_data/5_train_validation_test/X_validation_{}_1150.pickle".format(
        target_string), "wb") as f:
        pickle.dump(X_validation, f)
    with open("../../0_data/5_train_validation_test/X_test_{}_1150.pickle".format(
        target_string), "wb") as f:
        pickle.dump(X_test, f)
    with open("../../0_data/5_train_validation_test/y_train_{}_1150.pickle".format(
        target_string), "wb") as f:
        pickle.dump(y_train, f)
    with open("../../0_data/5_train_validation_test/y_validation_{}_1150.pickle".format(
        target_string), "wb") as f:
        pickle.dump(y_validation, f)
    with open("../../0_data/5_train_validation_test/y_test_{}_1150.pickle".format(
        target_string), "wb") as f:
        pickle.dump(y_test, f)

    X_train = pickle.load(open("../../0_data/5_train_validation_test/X_train_{}_1150.pickle".format(
        target_string), "rb"))
    X_validation = pickle.load(open("../../0_data/5_train_validation_test/X_validation_{}_1150.pickle".format(
        target_string), "rb"))
    X_test = pickle.load(open("../../0_data/5_train_validation_test/X_test_{}_1150.pickle".format(
        target_string), "rb"))
    y_train = pickle.load(open("../../0_data/5_train_validation_test/y_train_{}_1150.pickle".format(
        target_string), "rb"))
    y_validation = pickle.load(open("../../0_data/5_train_validation_test/y_validation_{}_1150.pickle".format(
        target_string), "rb"))
    y_test = pickle.load(open("../../0_data/5_train_validation_test/y_test_{}_1150.pickle".format(
        target_string), "rb"))

    logging.info("Starting {}".format(target_string))

    score, conf_mat, clf = train_model(X_train, y_train,
        X_validation, y_validation, target_string)

    with open("../../4_models/rf_{}_{}iterations_athreshold_{}_{}initial.pickle".format(
        target_string, iterations, step, num_samples), "wb") as f:
        pickle.dump(clf, f)

    logging.info("Initial model trained")
    logging.info("Step size: {}; Samples selected: {}; Iterations: {}".format(
        step, num_samples, iterations))
    logging.info("Size of train set: {}".format(X_train.shape[0]))
    logging.info("Number of 0: {}".format(
        y_train.shape[0] - y_train.sum()))
    logging.info("Number of 1: {}".format(y_train.sum()))
    logging.info("Initial score: {}".format(score))
    logging.info("Initial conf_matrix: {}".format(conf_mat))

    for i in range(iterations):
        # process the data
        tmp_df, feature_matrix_n, new_X_train_func = process_data_for_pseudolabel(
            random_seed=10191994+i, num_samples=step)

        for j in range(num_samples):
            # add first value that helps
            X_train_new, y_train_new = pseudolabel_data(
                X_train, y_train, target_string,
                clf, tmp_df, feature_matrix_n, new_X_train_func,
                threshold="adaptive")

            score_new, conf_mat_new, clf_new = train_model(
                X_train_new, y_train_new,
                X_validation, y_validation, target_string)

            if score_new >= score: # try adding = for now

                score = score_new
                conf_mat = conf_mat_new
                clf = clf_new

                X_train = X_train_new
                y_train = y_train_new

                logging.info("Size of train set: {}".format(X_train.shape[0]))
                logging.info("Number of 0: {}".format(
                    y_train.shape[0] - y_train.sum()))
                logging.info("Number of 1: {}".format(y_train.sum()))
                logging.info("Updated score: {}".format(score))
                logging.info("Updated conf_matrix: {}\n\n".format(conf_mat))
            else:
                logging.info("Not improved: {}\n\n".format(score_new))

        logging.info("Pseudolabel {} finished".format(i+1))

    return clf

# =========================================================================== #
# HELPER FUNCTIONS
# =========================================================================== #

def process_data_for_pseudolabel(random_seed=10191994, num_samples=1000):
    """Process a big chunk of random samples for pseudolabel procedure

    Args:
        random_seed - random seed for review selection
        num_samples - number of samples to mine

    Returns:
        tmp_df - random sample df
        feature_matrix_n - full feature_matrix
        new_X_train_func - new feature_matrix for random samples
    """
    tmp_df = select_random_reviews(random_seed, num_samples)

    tmp_df = tmp_df[tmp_df["review_id"].apply(
        lambda x: x not in handlabeled_set)].reset_index(
        ).drop("index", axis=1)

    # I think without the drop here, when producing the feature
    # matrix, because it takes into account index when doing
    # parallel processing, it will create the full number of
    # indices and replace missing ones with NA rows... or something...

    feature_matrix_n = produce_feature_matrix(tmp_df[["text"]])
    feature_matrix_n = feature_matrix_n.fillna(0)
    new_X_train_func = feature_matrix_n.iloc[:, 1:]

    return tmp_df, feature_matrix_n, new_X_train_func

def train_model(feature_matrix, target, validation_x, validation_y,
    target_string, roc=0):
    """Train a classifier on review data to get topic classes

    Args:
        feature_matrix - features
        target - target class data
        validation_x - validation features
        validation_y - validation target
        target_string - column name of intended target
        roc - whether to generate auc score or not

    Returns:
        accuracy scores, conf matrices, clf
    """
    clf = RandomForestClassifier(n_estimators=1000,
                                 max_depth=3,
                                 min_samples_leaf=10,
                                 max_features=0.6,
                                 random_state=10191994,
                                 class_weight="balanced",
                                 n_jobs=-1)

    try:
        score = clf.fit(feature_matrix[:, 1:], # because first column is raw text
                        target).score(
                        validation_x[:, 1:],
                        validation_y)

        conf_mat = confusion_matrix(validation_y,
            clf.predict(validation_x[:, 1:]))

        if roc:
            fpr, tpr, _ = roc_curve(validation_y,
                            clf.predict_proba(validation_x[:, 1:])[:,1])
            auc_score = auc(fpr, tpr)

        clf.fit(feature_matrix[:, 1:], target)
    except: # np array versus dataframe
        score = clf.fit(feature_matrix.iloc[:, 1:], # because first column is raw text
                        target).score(
                        validation_x[:, 1:],
                        validation_y)

        conf_mat = confusion_matrix(validation_y,
            clf.predict(validation_x[:, 1:]))

        if roc:
            fpr, tpr, _ = roc_curve(validation_y,
                            clf.predict_proba(validation_x[:, 1:])[:,1])
            auc_score = auc(fpr, tpr)

        clf.fit(feature_matrix.iloc[:, 1:], target)

    if roc:
        return score, conf_mat, auc_score, clf
    else:
        return score, conf_mat, clf

def pseudolabel_data(feature_matrix_o, target_o, target_string_o, clf,
    tmp_df, feature_matrix_n, new_X_train_func, threshold=0.9, num_to_add=1):
    """Adds in pseudolabeled data to training data if predicted
    above the threshold

    Args:
        feature_matrix_o - original features
        target_o - original target class data
        target_string_o - original column name of intended target
        clf - classifier object to use for labeling data
        tmp_df - random sample df
        feature_matrix_n - full feature matrix
        new_X_train_func - new feature matrix from sampled data
        threshold - threshold of prediction probability
        num_to_add - number of samples to add if threshold is adaptive

    Returns:
        new_feature_matrix - appended feature matrix
        new_target - appended target
    """
    pred_proba = clf.predict_proba(new_X_train_func)
    pred = clf.predict(new_X_train_func)

    # create new dataset
    confident_list_x = []
    confident_list_y = []

    # include code here to balance out set
    num_0 = target_o.shape[0] - target_o.sum()
    num_1 = target_o.sum()

    if threshold == "adaptive":
        ranked_indices = np.argsort(np.apply_along_axis(min, 1, pred_proba))
        # the logic behind this is that if you choose the most confident ones
        # it will essentially reinforce the features and separation that exists
        # but if you look at the ones that are hard to classify,
        # then perhaps the model can "learn" from the boundaries
        # and if it performs better on the validation, then it can learn
        # new features that would better predict the outcome

        # other extreme didn't work...so try something in the middle?
        # ranked_indices = np.argsort(np.apply_along_axis(lambda x: abs(x[0]-x[1]),
            # 1, pred_proba))
        ranked_pred_proba = pred_proba[ranked_indices]
        counter = 0

        for index, row in list(
            zip(ranked_indices, ranked_pred_proba))[500:]:

            if tmp_df["review_id"].iloc[index] not in handlabeled_set:

                num_0_plus = len([x for x in confident_list_y if x == 0])
                num_1_plus = len([x for x in confident_list_y if x == 1])

                # can make this an or
                if (int(num_0) + num_0_plus) <= (int(num_1) + num_1_plus) and (
                    row[0] > row[1]):

                    tmp = pd.DataFrame(feature_matrix_n.iloc[index]).transpose()
                    confident_list_x.append(tmp)
                    confident_list_y.append(pred[index])
                    handlabeled_set.add(tmp_df["review_id"].iloc[index])
                    counter += 1

                if (int(num_0) + num_0_plus) >= (int(num_1) + num_1_plus) and (
                    row[1] > row[0]):

                    tmp = pd.DataFrame(feature_matrix_n.iloc[index]).transpose()
                    confident_list_x.append(tmp)
                    confident_list_y.append(pred[index])
                    handlabeled_set.add(tmp_df["review_id"].iloc[index])
                    counter += 1

                if counter >= num_to_add:
                    logging.info("Row added: {}".format(row))
                    break

    else:
        for index, row in enumerate(pred_proba):

            if tmp_df["review_id"].iloc[index] not in handlabeled_set:

                num_0_plus = len([x for x in confident_list_y if x == 0])
                num_1_plus = len([x for x in confident_list_y if x == 1])

                if (int(num_0) + num_0_plus) <= (int(num_1) + num_1_plus) and (
                    row[0] >= threshold):

                    tmp = pd.DataFrame(feature_matrix_n.iloc[index]).transpose()
                    confident_list_x.append(tmp)
                    confident_list_y.append(pred[index])
                    handlabeled_set.add(tmp_df["review_id"].iloc[index])

                if (int(num_0) + num_0_plus) >= (int(num_1) + num_1_plus) and (
                    row[1] >= threshold):

                    tmp = pd.DataFrame(feature_matrix_n.iloc[index]).transpose()
                    confident_list_x.append(tmp)
                    confident_list_y.append(pred[index])
                    handlabeled_set.add(tmp_df["review_id"].iloc[index])

    try:
        new_x = pd.concat(confident_list_x, axis=0)
        new_x.columns = feature_matrix.columns
    except:
        new_x = pd.DataFrame()

    new_y = pd.DataFrame(confident_list_y)

    try: # they get transformed to numpy arrays after splitting
        feature_matrix_o_df = pd.DataFrame(feature_matrix_o)
        feature_matrix_o_df.columns = feature_matrix.columns

        new_X_train = pd.concat([feature_matrix_o_df,
                              new_x], axis=0)
        new_y_train = pd.concat([pd.DataFrame(target_o),
                          new_y])
    except:
        new_X_train = pd.concat([feature_matrix_o,
                              new_x], axis=0)
        new_y_train = pd.concat([target_o,
                          new_y])

    return new_X_train, new_y_train

def select_influential_reviews(random_seed, num_reviews):
    """Select a random review from the influential reviews json

    Args:
        random_seed - int random seed for random library
        num_reviews - int number of reviews to retrieve

    Returns:
        review_df - dataframe of randomly selected reviews from influential
    """
    random.seed(random_seed)

    with open(os.path.join("../../0_data/2_processed_data",
        "business_elite_subset_reviews.json"), "r") as f:
        data = ujson.load(f)

        random_indexes = random.sample(range(0, len(data)), num_reviews)

        review_dict = {}

        counter = 0

        for random_index in random_indexes:
            if data[random_index]["review_id"] not in handlabeled_set:
                review_dict[counter] = data[random_index]
                counter += 1

        out_df = pd.DataFrame.from_dict(review_dict, orient="index")
        return out_df

def select_random_reviews(random_seed, num_reviews):
    """Select a random number of reviews from the whole set
    copied over from select_random_reviews.py
    ~ 45s for 10000

    Args:
        random_seed - int random seed for random library
        num_reviews - int number of reviews to retrieve

    Returns:
        review_df - dataframe of randomly selected reviews
    """
    random.seed(random_seed)

    random_indexes = random.sample(range(0, max_review), num_reviews)

    review_dict = {}

    counter = 0

    for random_index in random_indexes:
        # select the right file to open
        tmp_row = review_count_data[((review_count_data["cum_sum"] -
            review_count_data["review_count"]) <=
            random_index + 1) & (random_index + 1 <=
            review_count_data["cum_sum"])]
        business_id = tmp_row[["business_id"]].iloc[0]["business_id"]
        file_index = (random_index
                      - (tmp_row["cum_sum"] - tmp_row["review_count"]) - 1)
        # open the file and retrieve the review
        with open(os.path.join(static_path,
            "reviews_by_business_id",
            "{}.json".format(business_id)), "r") as f:
            tmp_data = ujson.load(f)
            review_dict[counter] = tmp_data[file_index.iloc[0]]
            counter += 1

    out_df = pd.DataFrame.from_dict(review_dict, orient="index")
    return out_df

# =========================================================================== #
# RUN AND SAVE MODELS
# =========================================================================== #

clf_ambiance = train_pseudolabel_model(feature_matrix, target, "ambiance",
    iterations=4, step=5000, num_samples=200)
with open("../../4_models/rf_ambiance_4iterations_athreshold_5000_200.pickle", "wb") as f:
    pickle.dump(clf_ambiance, f)

clf_price = train_pseudolabel_model(feature_matrix, target, "price",
    iterations=4, step=5000, num_samples=200)
with open("../../4_models/rf_price_4iterations_athreshold_5000_200.pickle", "wb") as f:
    pickle.dump(clf_price, f)

clf_service = train_pseudolabel_model(feature_matrix, target, "service",
    iterations=4, step=5000, num_samples=200)
with open("../../4_models/rf_service_4iterations_athreshold_5000_200.pickle", "wb") as f:
    pickle.dump(clf_service, f)

clf_food = train_pseudolabel_model(feature_matrix, target, "food",
    iterations=4, step=5000, num_samples=200)
with open("../../4_models/rf_food_4iterations_athreshold_5000_200.pickle", "wb") as f:
    pickle.dump(clf_food, f)


import os

# file_path = "/home/hjm/sign/test/"
file_path = input("file_path: ")
# file_path = "/home/hjm/lenet_train/images/"
# anno_path = "/home/hjm/sign/testxml/"
anno_path = input("anno_path: ")
# anno_path = "/home/hjm/lenet_train/test/"

list = os.listdir(file_path)
list = sorted(list)

anno_list = os.listdir(anno_path)
anno_list = sorted(anno_list)
print(list)

file = ""

with open("a.txt",'w') as w:
    for l in list:
        # src = file_path + l
        l = l.strip(".png")+'\n'
        # file += l
        w.write(l)

w.close()
    # print(file)

with open('a.txt','r') as r:
    line = r.readlines()
    line = sorted(line)
    m = len(line)
    print(m)
# r.close()

i=0
while(m>0):
    for a in anno_list:
        src = os.path.join(anno_path,a)
        print(src)
        line[i]=line[i].strip()
        dst = anno_path + line[i] +".xml"
        os.rename(src,dst)
        i += 1
        m -= 1
r.close()
# import os
# import shutil
#
# file_path = '/home/hjm/lenet_train/t/'
# # file_path = '/home/hjm/lenet_train/test/'
# path = '/home/hjm/lenet_train/test/'
# # path = '/home/hjm/lenet_train/t/'
# list = os.listdir(file_path)
# list = sorted(list)
#
# i=0
# m=1768
# for l in list:
#     if m > 0:
#         src = os.path.join(file_path,l)
#         dst = os.path.join(path,l)
#         shutil.move(src, dst)
#     m -= 1
#     print(m)


import os

from jinja2 import Environment, PackageLoader, select_autoescape
from sanic import Sanic, request, response

from blur_detection_core import blur_check_file, blur_check_link
from custom_exceptions import (Base64ImageError, ImageInvalidError,
                               UrlInvalidError)

env = Environment(loader=PackageLoader('templates'),
                  autoescape=select_autoescape(['html', 'xml', 'tpl']),
                  enable_async=True)

UPLOAD_FOLDER = '/static/uploads/'

async def template(tpl, **kwargs):
    template = env.get_template(tpl)
    content = await template.render_async(kwargs)
    return response.html(content)

app = Sanic(__name__)
app.static('/static', './static')

@app.route("/")
async def test(req):
    return response.text("I\'m a teapot", status=418)

@app.route("/upload", methods=['GET', 'POST'])
async def upload(req):
    if req.method == 'POST':
        # check if there is a file in the request
        if 'file' not in req.files:
            return await template('upload.html', msg='No file selected')
        file = req.files['file'][0]
        # if no file is selected
        if file.name == '':
            return await template('upload.html', msg='No file selected')

        if file:
            with open(os.path.join(os.getcwd() + UPLOAD_FOLDER, file.name), "wb") as f:
                f.write(file.body)
            # file.save(os.path.join(os.getcwd() + UPLOAD_FOLDER, file.filename))
            blur_score = blur_check_file(file)
            # extract the text and display it
            return await template('upload.html',
                                  msg='Successfully processed',
                                  blur_score=blur_score,
                                  img_src=UPLOAD_FOLDER + file.name)
    elif req.method == 'GET':
        return await template('upload.html')

@app.route('/check', methods=['POST'])
async def image_check_api(req: request.Request):
    """Method to handle POST request to host/check.

    Arguments:
        request {sanic.request.Request} -- Incoming request.
    """
    if not req.json or 'url' not in req.json:
        return response.text('Error - no url', status=400)

    message = 'Good image'
    try:
        score = blur_check_link(req.json['url'])
    except (UrlInvalidError, ImageInvalidError, Base64ImageError) as e:
        message = str(e)
        score = -1

    return response.json({'score': score, 'message': message})

if __name__ == '__main__':
    app.run()


import numpy as np
import pandas as pd
import pbc
from pymatgen.core.periodic_table import Element
from sklearn.cluster import KMeans
import sys
import os
import glob
from joblib import Parallel, delayed

import warnings

CC_CUTOFF = 0.03
RADIUS_FACTOR = 1.2

def get_nbrs(crystal_xyz, crystal_lat, R_max):
    A = np.transpose(crystal_lat)
    B = np.linalg.inv(A)
    crystal_red = np.matmul(crystal_xyz, np.transpose(B))
    crystal_nbrs = pbc.get_shortest_distances(crystal_red, A, R_max,
                                              crdn_only=True)
    return crystal_nbrs

def get_radius(e):
    r = e.atomic_radius_calculated
    if r is None:
        r = e.atomic_radius
    if r is None:
        raise NameError('Not found the atomic radius for the element: {}'.format(e))
    else:
        return r

def get_nn_cluster(X):
    with warnings.catch_warnings():
        warnings.simplefilter("ignore")
        kmeans = KMeans(n_clusters=2, random_state=0).fit(X)
    cc_distance = abs(kmeans.cluster_centers_[0,0]-kmeans.cluster_centers_[1,0])
    if cc_distance > CC_CUTOFF:
        nnc_index = np.argmin(kmeans.cluster_centers_)
        nnc_nbrs = [i for i in range(len(X)) if kmeans.labels_[i] == nnc_index]
        return np.array(nnc_nbrs)
    else:
        return np.arange(len(X))

def get_nnc_loop(X):
    indices = np.arange(len(X))
    for _ in range(10):
        if len(indices) <= 1:
            break
        nnc = get_nn_cluster(X)
        if len(nnc) == len(indices):
            break
        else:
            indices = indices[nnc]
            X = X[nnc]
    return indices

def get_neighbors(geom):
    elems = [Element.from_Z(z) for z in geom.atomic_numbers]
    radii = np.array([get_radius(e) for e in elems])
    cutoff = radii[:,np.newaxis] + radii[np.newaxis, :]
    vol_atom = (4 * np.pi / 3) * np.array([r**3 for r in radii]).sum()
    factor_vol = (geom.volume / vol_atom)**(1.0/3.0)
    factor = factor_vol * RADIUS_FACTOR
    cutoff *= factor
    candidates = get_nbrs(geom.cart_coords, geom.lattice.matrix, cutoff)
    neighbors = []
    for j in range(len(candidates)):
        dists = []
        for nbr in candidates[j]:
            i = nbr[0]
            d = nbr[1]
            r = nbr[2]
            dists.append(d / cutoff[j,i])
        X = np.array(dists).reshape((-1, 1))
        nnc_nbrs = get_nnc_loop(X)
        neighbors.append([candidates[j][i][0] for i in nnc_nbrs])
    return neighbors

def get_structure(m):
    if m['nsites'] == len(m['structure']):
        return m['structure']
    else:
        s = m['structure'].get_primitive_structure()
        for _ in range(10):
            if m['nsites'] == len(s):
                return s
            else:
                s = s.get_primitive_structure()
        raise NameError('The primitive structure could not be got for {}'.format(m['material_id']))

def load_materials(filepath):
    try:
        data = np.load(filepath, allow_pickle=True)['materials']
    except UnicodeError:
        data = np.load(filepath, allow_pickle=True, encoding='latin1')['materials']
    return data

def process(data_path):
    materials = load_materials(data_path)
    material_ids = [m['material_id'] for m in materials]
    structures = [get_structure(m) for m in materials]
    data_ac = []
    data_nbr = []
    for geom in structures:
        neighbors = get_neighbors(geom)
        data_ac.append(geom.atomic_numbers)
        data_nbr.append(neighbors)
    graph_path = data_path.replace('mp_data', 'mp_graph')
    np.savez_compressed(graph_path, graph_names=material_ids,
                        graph_nodes=data_ac, graph_edges=data_nbr)

def main(data_dir, num_cpus):
    if not os.path.isdir(data_dir):
        print('Not found the data directory: {}'.format(data_dir))
        exit(1)

    data_files = sorted(glob.glob(os.path.join(data_dir, 'mp_data_*.npz')))
    Parallel(n_jobs=num_cpus, verbose=10)([delayed(process)(path) for path in data_files])

if __name__ == '__main__':
    import argparse

    parser = argparse.ArgumentParser(description='Crystal Graph Coordinator.')
    parser.add_argument('--data_dir', metavar='PATH', type=str, default='data',
                        help='The path to a data directory (default: data)')
    parser.add_argument('--num_cpus', metavar='N', type=int, default=-1,
                        help='The number of CPUs used for processing (default: -1)')
    options = vars(parser.parse_args())

    main(**options)


import xml.etree.ElementTree as ET
import uuid
import os
import time
import logging
import logging.handlers
import uuid

class ScriptLogger():
    def __init__(self):
        self.log_file_name = 'logs/latest.log'
        self.path = os.getcwd()
        self.logpath = os.path.join(self.path, 'logs')
        if not os.path.exists(self.logpath):
            os.mkdir(self.logpath)
        self.logging_level = logging.INFO
        # set TimedRotatingFileHandler for root
        self.formatter = logging.Formatter('%(asctime)s %(name)s %(levelname)s %(message)s')
        # use very short interval for this example, typical 'when' would be 'midnight' and no explicit interval
        self.handler = logging.handlers.TimedRotatingFileHandler(self.log_file_name, when="S", interval=30, backupCount=10)
        self.handler.setFormatter(self.formatter)
        self.logger = logging.getLogger() # or pass string to give it a name
        self.logger.addHandler(self.handler)
        self.logger.setLevel(self.logging_level)
    def getLogger(self):
        return self.logger

class DataFixtureCreator():
    def create_operator(self):
        id = str(uuid.uuid4())
        return {"model":"registry.operator", "pk":id, "fields":{"company_name":"", "website":"", "email":"", "phone_number":"", "operator_type":"", "address":"", "operational_authorizations":"", "authorized_activities":"", "vat_number":"", "insurance_number":"", "company_number":"", "country":""}}

    def create_owner(self):
        id = str(uuid.uuid4())
        return {"model":"registry.person", "pk":id, "fields":{}}

    def create_contact(self):
        id = str(uuid.uuid4())
        return {"model":"registry.contact", "pk":id, "fields":{"first_name":"", "middle_name":"", "last_name":"", "email":"", "phone_number":"", "identification_number":""}}

    def create_address(self):
        id = str(uuid.uuid4())
        return {"model":"registry.address", "pk":id, "fields":{"address_line_1":"", "address_line_2":"", "address_line_3":"", "postcode":"", "city":"", "country":""}}

    def create_manufacturer(self):
        id = str(uuid.uuid4())
        return {"model":"registry.address", "pk":id, "fields":{"full_name":"", "common_name":"", "address":"","acronym":"", "role":"", "country":""}}

    def create_type_certificate(self):
        id = str(uuid.uuid4())
        return {"model":"registry.typecertificate", "pk":id, "fields":{"type_certificate_id":"", "type_certificate_issuing_country":"", "type_certificate_holder":"", "type_certificate_holder_country":""}}

    def create_aircraft(self):
        id = str(uuid.uuid4())
        return {"model":"registry.aircraft", "pk":id, "fields":{"operator":"","mass":0,"is_airworthy":1, "make":"", "model":"","master_series":"","series":"","popular_name":"", "manufacturer":"", "category":"","registration_mark":"", "sub_category":"", "icao_aircraft_type_designator":"", "max_certified_takeoff_weight":0,"esn":0, "maci_number":"" , "status":1 }}

if __name__ == "__main__":
    myLogger = ScriptLogger()
    logger = myLogger.getLogger()
    myDataFixtureCreator = DataFixtureCreator()
    output_data = []

    # Global lookup for Aircraft
    TAG_DICT = {'IRCA_ID':"", "REGISTRATION_MARK":"registration_mark", "MANUFACTURER":"" ,"MAKE":"make", "MODEL":"model","SERIES1":"master_series", "SERIES2":"series","BAPTISM":"popular_name", "SER_NUM":"","OWNER_NAME":"", "OWNER_ADD_1":"address_line_1", "OWNER_ADD_2":"address_line_2", "OWNER_ADD_3":"address_line_3","OWNER_STATE":"", "OPERATOR_NAME":"", "OPERATOR_ADD_1":"address_line_1", "OPERATOR_ADD_2":"address_line_2", "OPERATOR_ADD_3":"address_line_3","OPERATOR_STATE":"", "AUTHORITY":"", "CONTACT":"", "NAT_REG":"", "REGISTRATION_DATE":"", "UPDATE_DATE":""}

    IGNORE_TAG_LIST = ['OWNER_ADD','OPERATOR_ADD']

    # root = ET.parse('icao.xml').getroot()
    root = ET.parse('ICAO.xml').getroot()

    search_keys = TAG_DICT.keys();
    for t in root.findall('.//AIRCRAFT/*'):
        address_tree = t.getchildren()

        if address_tree:
            address = myDataFixtureCreator.create_address()
            for child in address_tree:
                if child.tag not in search_keys and \
                    child.tag not in IGNORE_TAG_LIST:
                        logger.info("%s tag not in Tag Dict and IGNORE LIST" % child.tag)
                else:
                    # create JSON
                    second_level_children = child.getchildren()
                    if second_level_children:
                        # address entities
                        for cur_second_level_children in second_level_children:
                            # print(TAG_DICT[cur_second_level_children.tag], cur_second_level_children.text)
                            address['fields'][TAG_DICT[cur_second_level_children.tag]] = cur_second_level_children.text
                    else:
                        pass
            output_data.append(address)

        else:
            if not t.tag in search_keys:
                logger.info("%s tag not in Tag Dict" % t.tag)

            else:
                # pass
                print(t.tag, t.text)
                aircraft = myDataFixtureCreator.create_aircraft()
                # check if the keys exist

                # create JSON

    # print(output_data)


import logging
import requests
from time import sleep
from pathlib import Path
from requests_cache import CachedSession
from requests_cache.backends import SQLiteCache
from requests.exceptions import RequestException

from bbot.core.errors import WordlistError

log = logging.getLogger("bbot.core.helpers.web")

def wordlist(self, path, lines=None, **kwargs):
    if not path:
        raise WordlistError(f"Invalid wordlist: {path}")
    if not "cache_hrs" in kwargs:
        kwargs["cache_hrs"] = 720
    if self.is_url(path):
        filename = self.download(str(path), **kwargs)
        if filename is None:
            raise WordlistError(f"Unable to retrieve wordlist from {path}")
    else:
        filename = Path(path).resolve()
        if not filename.is_file():
            raise WordlistError(f"Unable to find wordlist at {path}")

    if lines is None:
        return filename
    else:
        lines = int(lines)
        with open(filename) as f:
            read_lines = f.readlines()
        cache_key = f"{filename}:{lines}"
        truncated_filename = self.cache_filename(cache_key)
        with open(truncated_filename, "w") as f:
            for line in read_lines[:lines]:
                f.write(line)
        return truncated_filename

def download(self, url, **kwargs):
    """
    Downloads file, returns full path of filename
    If download failed, returns None

    Caching supported via "cache_hrs"
    """
    success = False
    filename = self.cache_filename(url)
    cache_hrs = float(kwargs.pop("cache_hrs", -1))
    log.debug(f"Downloading file from {url} with cache_hrs={cache_hrs}")
    if cache_hrs > 0 and self.is_cached(url):
        log.debug(f"{url} is cached")
        success = True
    else:
        method = kwargs.get("method", "GET")
        try:
            with self.request(method=method, url=url, stream=True, raise_error=True, **kwargs) as response:
                status_code = getattr(response, "status_code", 0)
                log.debug(f"Download result: HTTP {status_code}")
                if status_code != 0:
                    response.raise_for_status()
                    with open(filename, "wb") as f:
                        for chunk in response.iter_content(chunk_size=8192):
                            f.write(chunk)
                    success = True
        except RequestException as e:
            log.warning(f"Failed to download {url}: {e}")
            return
        except AttributeError:
            return

    if success:
        return filename.resolve()

def request(self, *args, **kwargs):
    """
    Multipurpose function for making web requests

    Supports custom sessions
        session Request.Session()

    Arguments
        cache_for (Union[None, int, float, str, datetime, timedelta]): Cache response for <int> seconds
        raise_error (bool): Whether to raise exceptions (default: False)
    """

    raise_error = kwargs.pop("raise_error", False)

    cache_for = kwargs.pop("cache_for", None)
    if cache_for is not None:
        log.debug(f"Caching HTTP session with expire_after={cache_for}")
        try:
            session = self.cache_sessions[cache_for]
        except KeyError:
            db_path = str(self.cache_dir / "requests-cache.sqlite")
            backend = SQLiteCache(db_path=db_path)
            session = CachedSession(expire_after=cache_for, backend=backend)
            self.cache_sessions[cache_for] = session

    if kwargs.pop("session", None) or not cache_for:
        session = kwargs.pop("session", None)

    http_timeout = self.config.get("http_timeout", 20)
    user_agent = self.config.get("user_agent", "BBOT")

    # in case of URL only, assume GET request
    if len(args) == 1:
        kwargs["url"] = args[0]
        args = []

    url = kwargs.get("url", "")
    retries = kwargs.pop("retries", 0)

    if not args and "method" not in kwargs:
        kwargs["method"] = "GET"

    if not "timeout" in kwargs:
        kwargs["timeout"] = http_timeout

    headers = kwargs.get("headers", None)

    if headers is None:
        headers = {}
    if "User-Agent" not in headers:
        headers.update({"User-Agent": user_agent})
    kwargs["headers"] = headers

    http_debug = self.config.get("http_debug", False)
    while retries == "infinite" or retries >= 0:
        try:
            if http_debug:
                logstr = f"Web request: {str(args)}, {str(kwargs)}"
                log.debug(logstr)
            if session is not None:
                response = session.request(*args, **kwargs)
            else:
                response = requests.request(*args, **kwargs)
            if http_debug:
                log.debug(f"Web response: {response} (Length: {len(response.content)}) headers: {response.headers}")
            return response
        except RequestException as e:
            log.debug(f"Error with request: {e}")
            if retries != "infinite":
                retries -= 1
            if retries == "infinite" or retries >= 0:
                log.warning(f'Error requesting "{url}" ({e}), retrying...')
                sleep(1)
            else:
                if raise_error:
                    raise e

def api_page_iter(self, url, page_size=100, json=True, **requests_kwargs):
    page = 1
    offset = 0
    while 1:
        new_url = url.format(page=page, page_size=page_size, offset=offset)
        result = self.request(new_url, **requests_kwargs)
        try:
            if json:
                result = result.json()
            yield result
        except Exception:
            import traceback

            log.warning(f'Error in api_page_iter() for url: "{new_url}"')
            log.debug(traceback.format_exc())
            break
        finally:
            offset += page_size
            page += 1

def curl(self, *args, **kwargs):

    url = kwargs.get("url", "")

    if not url:
        log.debug("No URL supplied to CURL helper")
        return

    curl_command = ["curl", url, "-s"]

    raw_path = kwargs.get("raw_path", False)
    if raw_path:
        curl_command.append("--path-as-is")

    # respect global ssl verify settings
    ssl_verify = self.config.get("ssl_verify")
    if ssl_verify == False:
        curl_command.append("-k")

    headers = kwargs.get("headers", {})

    ignore_bbot_global_settings = kwargs.get("ignore_bbot_global_settings", False)

    if ignore_bbot_global_settings:
        log.debug("ignore_bbot_global_settings enabled. Global settings will not be applied")
    else:
        http_timeout = self.config.get("http_timeout", 20)
        user_agent = self.config.get("user_agent", "BBOT")

        if "User-Agent" not in headers:
            headers["User-Agent"] = user_agent

        # add the timeout
        if not "timeout" in kwargs:
            timeout = http_timeout

        curl_command.append("-m")
        curl_command.append(str(timeout))

    for k, v in headers.items():
        if type(v) == list:
            for x in v:
                curl_command.append("-H")
                curl_command.append(f"{k}: {x}")

        else:
            curl_command.append("-H")
            curl_command.append(f"{k}: {v}")

    post_data = kwargs.get("post_data", {})
    if len(post_data.items()) > 0:
        curl_command.append("-d")
        post_data_str = ""
        for k, v in post_data.items():
            post_data_str += f"&{k}={v}"
        curl_command.append(post_data_str.lstrip("&"))

    method = kwargs.get("method", "")
    if method:
        curl_command.append("-X")
        curl_command.append(method)

    cookies = kwargs.get("cookies", "")
    if cookies:

        curl_command.append("-b")
        cookies_str = ""
        for k, v in cookies.items():
            cookies_str += f"{k}={v}; "
        curl_command.append(f'{cookies_str.rstrip(" ")}')

    path_override = kwargs.get("path_override", None)
    if path_override:
        curl_command.append("--request-target")
        curl_command.append(f"{path_override}")

    head_mode = kwargs.get("head_mode", None)
    if head_mode:
        curl_command.append("-I")

    raw_body = kwargs.get("raw_body", None)
    if raw_body:
        curl_command.append("-d")
        curl_command.append(raw_body)

    output_bytes = self.run(curl_command, text=False).stdout
    output = self.smart_decode(output_bytes)
    return output


from sys import stdin, stdout

def main():
    tc = int(stdin.readline())
    for _ in range(tc):
        hasEdges = set()
        numEdges = 0
        while True:
            line = stdin.readline().strip()
            if line.startswith('*'):
                break
            numEdges += 1
            line = line.split(',')
            u = line[0][1]
            v = line[1][0]
            hasEdges.add(u)
            hasEdges.add(v)
        vertices = stdin.readline().strip('\n').split(',')
        cc = len(vertices) - numEdges
        acorn = len(vertices) - len(hasEdges)
        trees = cc - acorn
        stdout.write('There are {} tree(s) and {} acorn(s).\n'.format(trees, acorn))

main()