# General imports

import numpy as np

import pandas as pd

import os, warnings, datetime, math

import matplotlib.pyplot as plt

import seaborn as sns

from sklearn.preprocessing import LabelEncoder

warnings.filterwarnings('ignore')
########################### Helpers

#################################################################################

## -------------------

## Memory Reducer

# :df pandas dataframe to reduce size             # type: pd.DataFrame()

# :verbose                                        # type: bool

def reduce_mem_usage(df, verbose=True):

    numerics = ['int16', 'int32', 'int64', 'float16', 'float32', 'float64']

    start_mem = df.memory_usage().sum() / 1024**2

    for col in df.columns:

        col_type = df[col].dtypes

        if col_type in numerics:

            c_min = df[col].min()

            c_max = df[col].max()

            if str(col_type)[:3] == 'int':

                if c_min > np.iinfo(np.int8).min and c_max < np.iinfo(np.int8).max:

                    df[col] = df[col].astype(np.int8)

                elif c_min > np.iinfo(np.int16).min and c_max < np.iinfo(np.int16).max:

                    df[col] = df[col].astype(np.int16)

                elif c_min > np.iinfo(np.int32).min and c_max < np.iinfo(np.int32).max:

                    df[col] = df[col].astype(np.int32)

                elif c_min > np.iinfo(np.int64).min and c_max < np.iinfo(np.int64).max:

                    df[col] = df[col].astype(np.int64)

            else:

                if c_min > np.finfo(np.float16).min and c_max < np.finfo(np.float16).max:

                    df[col] = df[col].astype(np.float16)

                elif c_min > np.finfo(np.float32).min and c_max < np.finfo(np.float32).max:

                    df[col] = df[col].astype(np.float32)

                else:

                    df[col] = df[col].astype(np.float64)

    end_mem = df.memory_usage().sum() / 1024**2

    if verbose: print('Mem. usage decreased to {:5.2f} Mb ({:.1f}% reduction)'.format(end_mem, 100 * (start_mem - end_mem) / start_mem))

    return df

## -------------------
########################### DATA LOAD

#################################################################################

print('Load Data')

train_df = pd.read_csv('../input/train_transaction.csv')

test_df = pd.read_csv('../input/test_transaction.csv')

test_df['isFraud'] = 0

train_identity = pd.read_csv('../input/train_identity.csv')

test_identity = pd.read_csv('../input/test_identity.csv')
########################### Base check

#################################################################################

for df in [train_df, test_df, train_identity, test_identity]:

    original = df.copy()

    df = reduce_mem_usage(df)

    for col in list(df):

        if df[col].dtype!='O':

            if (df[col]-original[col]).sum()!=0:

                df[col] = original[col]

                #print('Bad transformation', col)
train_df.columns
train_identity.head()
temp_df = pd.concat([train_df, test_df])

percent_missing = (temp_df.isnull().mean() * 100)

print(percent_missing.sort_values(ascending=False))
### Removing columns greater than a certain threshold

#thresh = 90

#reduced_col = (percent_missing[percent_missing<thresh])

#reduced_col.sort_values(ascending=False)
# numnerical columns

num_cols = train_df._get_numeric_data().columns

print('Total number of numerical columns are ', len(num_cols))

print(num_cols)

print('-------------------------------------------------------------------------------')

# categorical columns

cat_cols = set(train_df.columns) - set(num_cols)

print('Total number of categorical columns are', len(cat_cols))

print(cat_cols)
# Plotting categorical features

def make_categorical_plots(Vs):

    col = 4

    row = len(Vs)//col+1

    fig = plt.figure(figsize=(20,row*5))

    for i,v in enumerate(Vs):

        ax = plt.subplot(row,col,i+1)

        g1 = sns.barplot(x=v, y="isFraud", data=train_df, ax = ax)

        g1.legend()

        plt.title(v+" barplot w.r.t target", fontsize=16)

        g1.set_xlabel(v+ " values", fontsize=16)

        g1.set_ylabel("Probability", fontsize=16)

        #plt.title('Column: '+ str(v), fontsize=16)

        plt.subplots_adjust(hspace = 0.5)

    plt.show()
# Plotting desity plot for numerical features

def make_density_plots(Vs):

    col = 2

    row = len(Vs)//col+1

    fig = plt.figure(figsize=(20,row*5))

    for i,v in enumerate(Vs):

        ax = plt.subplot(row,col,i+1)

        g1 = sns.distplot(train_df[train_df['isFraud'] == 1][v].dropna(), label='Fraud',

                          ax=ax )

        g1 = sns.distplot(train_df[train_df['isFraud'] == 0][v].dropna(), label='NoFraud',

                              ax=ax)

        g1.legend()

        plt.title(v+" values distribution w.r.t target", fontsize=16)

        g1.set_xlabel(v+ " values", fontsize=16)

        g1.set_ylabel("Probability", fontsize=16)

        #plt.title('Column: '+ str(v), fontsize=16)

        plt.subplots_adjust(hspace = 0.5)

    plt.show()
# Function to plot histogram

def make_histogram_plots(Vs):

    col = 4

    row = len(Vs)//4+1

    plt.figure(figsize=(20,row*5))

    for i,v in enumerate(Vs):

        #print(v)

        plt.subplot(row,col,i+1)

        plt.title('Column: '+ str(v))

        h = plt.hist(train_df[v],bins=100)

        if len(h[0])>1: plt.ylim((0,np.sort(h[0])[-2]))

    plt.show()
import re
total_col = train_df.columns

# Finding all columns starting with 'card'

r = re.compile("^card")

card_list = list(filter(r.match, total_col)) # Read Note

print(card_list)
make_density_plots(['card1', 'card2', 'card3','card5'])
# Finding all columns starting with 'addr'

r = re.compile("^addr")

addr_list = list(filter(r.match, total_col)) # Read Note

print(addr_list)
make_density_plots(addr_list)
# Finding all columns starting with 'V'

r = re.compile("^V")

V_list = list(filter(r.match, total_col)) # Read Note

print(V_list)
# Plotting first 10 values

make_density_plots(['V1', 'V2', 'V3', 'V4', 'V5', 'V6', 'V7', 'V8', 'V9', 'V10'])
# col_corr = set() # Set of all the names of deleted columns

corr_matrix = train_df[V_list].corr()
####### Removing multi-correlated columns and retaining only that column which has high unique value,

#### This conveys that it has high variance, and may be has high impact on the model

#### Here, first we are making correlation matrix and then finding the correlations of columns with other columns

#### which are highly correlated. Then removing all other columns except for the columns which has highest unique value.

col_corr = set()

threshold = 0.80

for i in range(len(corr_matrix.columns)):

    high_corr = []

    high_corr.append(corr_matrix.columns[i])

    for j in range(i):

        mx = 0

        vx = corr_matrix.columns[i]

        #print('vx', vx)

        if (corr_matrix.iloc[i, j] >= threshold) and (corr_matrix.columns[j] not in col_corr):

            high_corr.append(corr_matrix.columns[j])

    if len(high_corr)>1:

        #print('high_corr',high_corr )

        for col in high_corr:

            n = train_df[col].nunique()

            if n>mx:

                mx = n

                vx = col

        #print('vx', vx)

        high_corr.remove(str(vx))

        col_corr.update(set(high_corr))

        #print('col_corr', col_corr)
print(list(col_corr))
# V_list_ = [x for x in V_list if x not in col_corr]

# print("Length of new V_list is", len(V_list_))

# print(V_list_)
# with threshold value of .80

# This is the output after finding the features which are highly correlated.

# You can find how to get these columns 3 cells above

V_list_ = ['V1', 'V3', 'V5', 'V7', 'V9', 'V13', 'V20', 'V24', 'V26', 'V28', 'V30', 'V36', 'V38', 'V45',

           'V47', 'V54', 'V55', 'V56', 'V62', 'V67', 'V76', 'V78', 'V83', 'V87', 'V88', 'V107', 'V109',

           'V110', 'V113', 'V115', 'V116', 'V119', 'V121', 'V122', 'V125', 'V138', 'V140', 'V142',

           'V147', 'V158', 'V160', 'V162', 'V166', 'V169', 'V174', 'V185', 'V198', 'V201', 'V209',

           'V210', 'V220', 'V221', 'V223', 'V239', 'V240', 'V241', 'V251', 'V252', 'V260', 'V262',

           'V267', 'V271', 'V281', 'V282', 'V283', 'V286', 'V289', 'V291', 'V301', 'V303', 'V305',

           'V307', 'V310', 'V325', 'V339']
#make_density_plots(['V_list_'])
# Plotting values

make_density_plots(['TransactionAmt'])
# Finding all columns starting with 'D'

r = re.compile("^D")

D_list = list(filter(r.match, total_col)) # Read Note

print(D_list)
make_density_plots(D_list)
# Finding all columns starting with 'M'

r = re.compile("^M")

M_list = list(filter(r.match, total_col)) # Read Note

print(M_list)
make_categorical_plots(['M1','M2','M3','M4','M5','M6','M7','M8','M9'])
make_categorical_plots(['addr1', 'addr2'])
train_identity.head()
train_identity.columns
temp_identity_df = pd.concat([train_identity, test_identity])

percent_missing = (temp_identity_df.isnull().mean() * 100)

print(percent_missing.sort_values(ascending=False))
# numnerical columns

num_identity_cols = train_identity._get_numeric_data().columns

print('Total number of numerical columns are ', len(num_identity_cols))

print(num_identity_cols)

print('-------------------------------------------------------------------------------')

# categorical columns

cat_identity_cols = set(train_identity.columns) - set(num_identity_cols)

print('Total number of categorical columns are', len(cat_identity_cols))

print(cat_identity_cols)
train_df['TransactionAmt'] / train_df.groupby(['card1'])['TransactionAmt'].transform('mean')
train_df.head()
# Find the groups

#train_df.groupby(['card1']).groups
sns.countplot(x="ProductCD", data=train_df)
sns.countplot(x="M4", data=train_df)
train_df['card1'].nunique()
# Freq encoding

i_cols = ['card1','card2','M4','D9',

          'addr1','addr2','dist1','dist2',

          'P_emaildomain', 'R_emaildomain'

         ]

for col in i_cols:

    temp_df = pd.concat([train_df[[col]], test_df[[col]]])

    valid_card = temp_df[col].value_counts().to_dict()

    train_df[col+'_fq_enc'] = train_df[col].map(valid_card)

    test_df[col+'_fq_enc']  = test_df[col].map(valid_card)
(train_df['card1'].astype(str)+'_'+train_df['card2'].astype(str)).nunique()
len(train_df)
TARGET = 'isFraud'

for col in ['card1','card2','addr1','addr2','M4']:

    temp_dict = train_df.groupby([col])[TARGET].agg(['mean']).reset_index().rename(

                                                        columns={'mean': col+'_target_mean'})

    temp_dict.index = temp_dict[col].values

    temp_dict = temp_dict[col+'_target_mean'].to_dict()

    train_df[col+'_target_mean'] = train_df[col].map(temp_dict)

    test_df[col+'_target_mean']  = test_df[col].map(temp_dict)
i_cols = V_list_

for df in [train_df, test_df]:

    df['V_sum'] = df[i_cols].sum(axis=1).astype(np.int8)

    df['V_na'] = df[i_cols].isna().sum(axis=1).astype(np.int8)
# Binary encoding for M columns

i_cols = ['M1','M2','M3','M5','M6','M7','M8','M9']

for df in [train_df, test_df]:

    df['M_sum'] = df[i_cols].sum(axis=1).astype(np.int8)

    df['M_na'] = df[i_cols].isna().sum(axis=1).astype(np.int8)
# Let's add some kind of client uID based on cardID ad addr columns

# The value will be very specific for each client so we need to remove it

# from final feature. But we can use it for aggregations.

train_df['uid'] = train_df['card1'].astype(str)+'_'+train_df['card2'].astype(str)

test_df['uid'] = test_df['card1'].astype(str)+'_'+test_df['card2'].astype(str)

train_df['uid2'] = train_df['uid'].astype(str)+'_'+train_df['addr1'].astype(str)+'_'+train_df['addr2'].astype(str)

test_df['uid2'] = test_df['uid'].astype(str)+'_'+test_df['addr1'].astype(str)+'_'+test_df['addr2'].astype(str)

train_df['uid3'] = train_df['uid2'].astype(str)+'_'+train_df['P_emaildomain'].astype(str)+'_'+train_df['R_emaildomain'].astype(str)

test_df['uid3'] = test_df['uid2'].astype(str)+'_'+test_df['P_emaildomain'].astype(str)+'_'+test_df['R_emaildomain'].astype(str)

# Check if the Transaction Amount is common or not (we can use freq encoding here)

# In our dialog with a model we are telling to trust or not to these values

train_df['TransactionAmt_check'] = np.where(train_df['TransactionAmt'].isin(test_df['TransactionAmt']), 1, 0)

test_df['TransactionAmt_check']  = np.where(test_df['TransactionAmt'].isin(train_df['TransactionAmt']), 1, 0)

i_cols = ['card1','card2','uid','uid2','uid3']

for col in i_cols:

    for agg_type in ['mean','std']:

        new_col_name = col+'_TransactionAmt_'+agg_type

        temp_df = pd.concat([train_df[[col, 'TransactionAmt']], test_df[[col,'TransactionAmt']]])

        #temp_df['TransactionAmt'] = temp_df['TransactionAmt'].astype(int)

        temp_df = temp_df.groupby([col])['TransactionAmt'].agg([agg_type]).reset_index().rename(

                                                columns={agg_type: new_col_name})

        temp_df.index = list(temp_df[col])

        temp_df = temp_df[new_col_name].to_dict()

        train_df[new_col_name] = train_df[col].map(temp_df)

        test_df[new_col_name]  = test_df[col].map(temp_df)
START_DATE = datetime.datetime.strptime('2017-11-30', '%Y-%m-%d')

for df in [train_df, test_df]:

    # Temporary variables for aggregation

    df['DT'] = df['TransactionDT'].apply(lambda x: (START_DATE + datetime.timedelta(seconds = x)))

    df['DT_M'] = ((df['DT'].dt.year-2017)*12 + df['DT'].dt.month).astype(np.int8)

    df['DT_W'] = ((df['DT'].dt.year-2017)*52 + df['DT'].dt.weekofyear).astype(np.int8)

    df['DT_D'] = ((df['DT'].dt.year-2017)*365 + df['DT'].dt.dayofyear).astype(np.int16)

    df['DT_hour'] = (df['DT'].dt.hour).astype(np.int8)

    df['DT_day_week'] = (df['DT'].dt.dayofweek).astype(np.int8)

    df['DT_day_month'] = (df['DT'].dt.day).astype(np.int8)
df[['DT','DT_M','DT_W', 'DT_D','DT_hour', 'DT_day_week', 'DT_day_month' ]].head()
# Total transactions per timeblock

for col in ['DT_M','DT_W','DT_D', 'DT_hour', 'DT_day_week', 'DT_day_month']:

    temp_df = pd.concat([train_df[[col]], test_df[[col]]])

    fq_encode = temp_df[col].value_counts().to_dict()

    train_df[col+'_total'] = train_df[col].map(fq_encode)

    test_df[col+'_total']  = test_df[col].map(fq_encode)

train_df[['DT_M_total','DT_W_total','DT_D_total', 'DT_hour_total', 'DT_day_week_total', 'DT_day_month_total']].head()
V_col_remove = [col for col in V_list if col not in V_list_]

#print(V_col_remove)
for col in ['M1','M2','M3','M5','M6','M7','M8','M9']:

    train_df[col] = train_df[col].map({'T':1, 'F':0})

    test_df[col]  = test_df[col].map({'T':1, 'F':0})
train_df.dtypes
train_df['ProductCD'].head()

#.fillna('unseen_before_label')
train_df['ProductCD'].isnull().sum()
train_df['ProductCD'].count()
train_df['ProductCD'].fillna('unseen_before_label').head()
train_df.select_dtypes(include=['object'])
for col in list(train_df):

    if train_df[col].dtype=='object':

        print(col)
# Frequency encoding for categorical column and then doing categorical encoding

for col in list(train_df):

    if train_df[col].dtype=='object':

        print(col)

        train_df[col] = train_df[col].fillna('NaN')

        test_df[col]  = test_df[col].fillna('NaN')

        train_df[col] = train_df[col].astype(str)

        test_df[col] = test_df[col].astype(str)

        le = LabelEncoder()

        le.fit(list(train_df[col])+list(test_df[col]))

        train_df[col] = le.transform(train_df[col])

        test_df[col]  = le.transform(test_df[col])

        train_df[col] = train_df[col].astype('category')

        test_df[col] = test_df[col].astype('category')
train_df['R_emaildomain']
rm_cols = [

    'uid','uid2','uid3',

    'DT','DT_M', 'DT_W', 'DT_D','DT_hour', 'DT_day_week', 'DT_day_month' # Already we have considered these

]

rm_cols.extend(V_col_remove)

#rm_cols
features_columns = [col for col in list(train_df) if col not in rm_cols]
len(train_df)
len(test_df)
features_columns.remove('isFraud')

features_columns
########################### Model

import lightgbm as lgb

from sklearn.model_selection import StratifiedKFold

import gc

SEED = 10

def make_predictions(train_df, test_df, features_columns, target, lgb_params, NFOLDS=3):

    N_SPLITS = 10

    folds = StratifiedKFold(n_splits=N_SPLITS, shuffle=True, random_state=SEED)

    # Main Data

    X,y = train_df[features_columns], train_df[TARGET]

    # Test Data and expport DF

    P,P_y  = test_df[features_columns], test_df[TARGET]

    test_df = test_df[['TransactionID',target]]

    predictions = np.zeros(len(test_df))

    for fold_, (trn_idx, val_idx) in enumerate(folds.split(X, y, groups=y)):

        print('Fold:',fold_+1)

        tr_x, tr_y = X.iloc[trn_idx,:], y[trn_idx]

        vl_x, vl_y = X.iloc[val_idx,:], y[val_idx]

        train_data = lgb.Dataset(tr_x, label=tr_y)

        #valid_data = lgb.Dataset(vl_x, label=v_y)

        if LOCAL_TEST:

            valid_data = lgb.Dataset(P, label=P_y)

        else:

            valid_data = lgb.Dataset(vl_x, label=vl_y)

        estimator = lgb.train(

                lgb_params,

                train_data,

                valid_sets = [train_data, valid_data],

                verbose_eval = 1000,

            )

        pp_p = estimator.predict(P)

        predictions += pp_p/NFOLDS

        if LOCAL_TEST:

            feature_imp = pd.DataFrame(sorted(zip(estimator.feature_importance(),X.columns)), columns=['Value','Feature'])

            print(feature_imp)

       # del tr_x, tr_y, vl_x, vl_y, train_data, valid_data

       # gc.collect()

    test_df['prediction']  = predictions

    return test_df
LOCAL_TEST = False

# Model params

from sklearn import metrics

lgb_params = {

                    'objective':'binary',

                    'boosting_type':'gbdt',

                    'metric':'auc',

                    'n_jobs':-1,

                    'learning_rate':0.01,

                    'num_leaves': 2**8,

                    'max_depth':-1,

                    'tree_learner':'serial',

                    'colsample_bytree': 0.7,

                    'subsample_freq':1,

                    'subsample':0.7,

                    'n_estimators':800,

                    'max_bin':255,

                    'verbose':-1,

                    'seed': SEED,

                    'early_stopping_rounds':100,

                }

# Model Train

if LOCAL_TEST:

    lgb_params['learning_rate'] = 0.01

    lgb_params['n_estimators'] = 20000

    lgb_params['early_stopping_rounds'] = 100

    test_predictions = make_predictions(train_df, test_df, features_columns, TARGET, lgb_params)

    print(metrics.roc_auc_score(test_predictions[TARGET], test_predictions['prediction']))

else:

    lgb_params['learning_rate'] = 0.01

    lgb_params['n_estimators'] = 800

    lgb_params['early_stopping_rounds'] = 100

    test_predictions = make_predictions(train_df, test_df, features_columns, TARGET, lgb_params, NFOLDS=2)
# submission

test_predictions['isFraud'] = test_predictions['prediction']

test_predictions[['TransactionID','isFraud']].to_csv('submission.csv', index=False)


from typing import List
'''
给你一个整型数组 nums ，在数组中找出由三个数组成的最大乘积，并输出这个乘积。

示例 1：

输入：nums = [1,2,3]
输出：6
示例 2：

输入：nums = [1,2,3,4]
输出：24
示例 3：

输入：nums = [-1,-2,-3]
输出：-6

来源：力扣（LeetCode）
链接：https://leetcode-cn.com/problems/maximum-product-of-three-numbers
著作权归领扣网络所有。商业转载请联系官方授权，非商业转载请注明出处。
'''

class Solution:
    def maximumProduct(self, nums: List[int]):
        nums.sort()
        print(nums)
        # 全正数 或者 全负数
        res1 = nums[-1]*nums[-2]*nums[-3]
        # 有正数有负数
        res2 = nums[0]*nums[1]*nums[-1]
        return max(res1, res2)

obj = Solution()
nums = [-1, -2, -3, -4]
print(obj.maximumProduct(nums))

628-三个数的最大乘积.py


from collections import defaultdict

paths = []

with open("spaths.txt") as f:
    pathss = f.readlines()
    for path in pathss:
        paths.append(path.strip().split("->"))

def get_path(src, dst):
    for path in paths:
        if src == path[0] and dst == path[-1]:
            return path

def common_sw(path1, path2):
    num_common_sw = 0
    for sw1, sw2 in zip(path1, path2):
        if sw1 == sw2:
            num_common_sw += 1

    return num_common_sw

results = defaultdict()
for src_sv1_id in range(1, 5):
    for dst_sv1_id in range(5, 9):
        src_sw1_id = f"sw{(src_sv1_id - 1) // 2 + 13}"
        dst_sw1_id = f"sw{(dst_sv1_id - 1) // 2 + 13}"
        path1 = get_path(src_sw1_id, dst_sw1_id)
        for src_sv2_id in range(1, 5):
            if src_sv1_id == src_sv2_id:
                continue
            for dst_sv2_id in range(5, 9):
                if dst_sv1_id == dst_sv2_id:
                    continue
                src_sw2_id = f"sw{(src_sv2_id - 1) // 2 + 13}"
                dst_sw2_id = f"sw{(dst_sv2_id - 1) // 2 + 13}"
                path2 = get_path(src_sw2_id, dst_sw2_id)
                num_common_sw = common_sw(path1, path2)
                results[f"sv{src_sv1_id}->sv{dst_sv1_id}/sv{src_sv2_id}->sv{dst_sv2_id}"] = num_common_sw

results = {k: v for k, v in sorted(results.items(), key=lambda item: item[1])}
with open("num_common_sw_sp.txt", "w") as f:
    for k, v in results.items():
        f.write(f"{k} {v}\n")


import time
import os
import sys
import math
import ctypes
import random
import datetime
from functools import reduce
from collections import deque
from collections import defaultdict
from itertools import chain
import logging
import socket

import popart
import numpy as np
from torch.utils.tensorboard import SummaryWriter

from bert_model import ExecutionMode, get_model, BertConfig
from bert_data import get_pretraining_dataset, get_squad_dataset
from bert_tf_loader import load_initializers_from_tf
from bert_optimizer import ScheduledOptimizerFactory
import utils

logger = logging.getLogger('BERT')

so_path = os.path.join(os.path.dirname(os.path.realpath(__file__)),
                       "custom_ops.so")
if os.path.exists(so_path):
    ctypes.cdll.LoadLibrary(so_path)
else:
    logger.warning("Could not find custom_ops.so. Execute `make` before running this script.")

def set_library_seeds(seed):
    np.random.seed(seed)
    random.seed(seed)

def bert_config_from_args(args):
    return BertConfig(**{k: getattr(args, k)
                         for k in BertConfig._fields if hasattr(args, k)})

def bert_add_embedding_inputs(args, model, sequence_info):
    if args.host_embedding == "NONE":
        indices = model.builder.addInputTensor(sequence_info, "indices")
        positions = model.builder.addInputTensor(sequence_info, "positions")
    else:   # "ALL", "WORD", "MERGE"
        expanded_sequence_info = popart.TensorInfo(
            "FLOAT16", [args.batch_size * args.sequence_length, args.hidden_size])
        indices = model.builder.addInputTensor(
            expanded_sequence_info, "indices_expanded")
        if args.host_embedding == "ALL":
            positions = model.builder.addInputTensor(
                expanded_sequence_info, "pos_expanded")
        else:
            positions = model.builder.addInputTensor(sequence_info, "positions")

    return indices, positions

def bert_add_inputs(args, model):
    sequence_info = popart.TensorInfo(
        "UINT32", [args.batch_size * args.sequence_length])
    indices, positions = bert_add_embedding_inputs(args, model, sequence_info)

    segments = model.builder.addInputTensor(sequence_info, "segments")
    labels = []
    masks = []
    mask_info = popart.TensorInfo("UINT32", [args.batch_size, 1])
    if args.task == "PRETRAINING":
        masks.append(
            model.builder.addInputTensor(mask_info, "mask_tokens_mask_idx"))
        masks.append(
            model.builder.addInputTensor(mask_info, "sequence_mask_idx"))
        mlm_info = popart.TensorInfo(
            "UINT32", [args.batch_size, args.mask_tokens])
        labels.append(model.builder.addInputTensor(mlm_info, "mask_labels"))
        nsp_info = popart.TensorInfo(
            "UINT32", [args.batch_size])
        labels.append(model.builder.addInputTensor(nsp_info, "nsp_labels"))
    elif args.task == "SQUAD":
        masks.append(model.builder.addInputTensor(mask_info, "seq_pad_idx"))
        if not args.inference:
            labels_info = popart.TensorInfo(
                "UINT32", [args.batch_size])
            labels.append(model.builder.addInputTensor(
                labels_info, "start_labels"))
            labels.append(model.builder.addInputTensor(
                labels_info, "end_labels"))
    return indices, positions, segments, masks, labels

def bert_logits_graph(model, indices, positions, segments, masks, mode):
    if mode == ExecutionMode.PHASED:
        logits = model(indices, positions, segments, masks)
    else:
        logits = model.build_graph(indices, positions, segments, masks)
    return logits

def bert_perplexity_graph(model, logits, labels):
    with model.mlm_scope:
        mlm_probs = model.builder.aiOnnx.softmax(
            [logits[0]], axis=2, debugPrefix="Softmax")

    losses = bert_loss_graph(model, [mlm_probs], [labels[0]])

    # Due to limitations on the way loss is handled with Popart, we'll anchor the loss and calculate
    # the perplexity on the host in the stats function.
    return losses

def bert_infer_graph(model, logits, include_probs=True):
    # NOTE: include_probs added as we don't need to calculate the softmax if we only care about accuracy and not
    # about loss
    probs = None
    if model.config.task == "SQUAD":
        if model.config.execution_mode == ExecutionMode.PHASED:
            with model.scope_provider(model.builder, model.squad_scope):
                predictions = list(
                    model.builder.aiOnnx.argmax([logit],
                                                axis=1,
                                                keepdims=0,
                                                debugPrefix=f"{logit}/ArgMax")
                    for logit in logits)
                if include_probs:
                    probs = list(
                        model.builder.aiOnnx.softmax(
                            [logit], axis=1, debugPrefix=f"{logit}/Softmax")
                        for logit in logits)

                    for prob in probs:
                        model.builder.setInplacePreferences(
                            prob, {"SoftmaxInplace": -1})
        else:
            with model.squad_scope:
                predictions = list(
                    model.builder.aiOnnx.argmax([logit],
                                                axis=1,
                                                keepdims=0,
                                                debugPrefix=f"{logit}/ArgMax")
                    for logit in logits)
                probs = list(
                    model.builder.aiOnnx.softmax(
                        [logit], axis=1, debugPrefix=f"{logit}/Softmax")
                    for logit in logits)
                for prob in probs:
                    model.builder.setInplacePreferences(
                        prob, {"SoftmaxInplace": -1})
    elif model.config.task == "PRETRAINING":
        if model.config.execution_mode == ExecutionMode.PHASED:
            with model.scope_provider(model.builder, model.nsp_scope):
                nsp_predictions = model.builder.aiOnnx.argmax(
                    [logits[1]], axis=1, keepdims=0, debugPrefix="ArgMax")
                if include_probs:
                    nsp_probs = model.builder.aiOnnx.softmax([logits[1]],
                                                             axis=1,
                                                             debugPrefix="Softmax")
            with model.scope_provider(model.builder, model.mlm_scope):
                mlm_predictions = model.builder.aiOnnx.argmax(
                    [logits[0]], axis=2, keepdims=0, debugPrefix="ArgMax")
                if include_probs:
                    mlm_probs = model.builder.aiOnnx.softmax([logits[0]],
                                                             axis=2,
                                                             debugPrefix="Softmax")
        else:
            with model.nsp_scope:
                nsp_predictions = model.builder.aiOnnx.argmax(
                    [logits[1]], axis=1, keepdims=0, debugPrefix="ArgMax")
                if include_probs:
                    nsp_probs = model.builder.aiOnnx.softmax([logits[1]],
                                                             axis=1,
                                                             debugPrefix="Softmax")
            with model.mlm_scope:
                mlm_predictions = model.builder.aiOnnx.argmax(
                    [logits[0]], axis=2, keepdims=0, debugPrefix="ArgMax")
                if include_probs:
                    mlm_probs = model.builder.aiOnnx.softmax([logits[0]],
                                                             axis=2,
                                                             debugPrefix="Softmax")
        predictions = [mlm_predictions, nsp_predictions]
        if include_probs:
            probs = [mlm_probs, nsp_probs]
    return predictions, probs

def bert_loss_graph(model, probs, labels):
    def loss(prob, label):
        if model.config.execution_mode == ExecutionMode.PHASED:
            if model.config.task == "SQUAD":
                with model.scope_provider(model.builder, model.squad_scope):
                    nllloss = model.builder.aiGraphcore.nllloss(
                        [prob, label],
                        reduction=popart.ReductionType.NoReduction,
                        debugPrefix=f"{label}/loss")

            elif 'nsp' in label:
                with model.scope_provider(model.builder, model.nsp_scope):
                    nllloss = model.builder.aiGraphcore.nllloss(
                        [prob, label],
                        reduction=popart.ReductionType.NoReduction,
                        ignoreIndex=2,
                        debugPrefix=f"{label}/loss")
            else:
                with model.scope_provider(model.builder, model.mlm_scope):
                    nllloss = model.builder.aiGraphcore.nllloss(
                        [prob, label],
                        reduction=popart.ReductionType.NoReduction,
                        ignoreIndex=0,
                        debugPrefix=f"{label}/loss")
        else:
            if model.config.task == "SQUAD":
                with model.squad_scope:
                    nllloss = model.builder.aiGraphcore.nllloss(
                        [prob, label],
                        reduction=popart.ReductionType.NoReduction,
                        debugPrefix=f"{label}/loss")
            elif 'nsp' in label:
                with model.nsp_scope:
                    nllloss = model.builder.aiGraphcore.nllloss(
                        [prob, label],
                        reduction=popart.ReductionType.NoReduction,
                        ignoreIndex=2,
                        debugPrefix=f"{label}/loss")
            else:
                with model.mlm_scope:
                    nllloss = model.builder.aiGraphcore.nllloss(
                        [prob, label],
                        reduction=popart.ReductionType.NoReduction,
                        ignoreIndex=0,
                        debugPrefix=f"{label}/loss")

        return nllloss

    return [loss(*p_l) for p_l in zip(probs, labels)]

def bert_add_logit_outputs(model, logits):
    outputs = {}
    for logit in logits:
        outputs[logit] = popart.AnchorReturnType("ALL")
    for out in outputs.keys():
        model.builder.addOutputTensor(out)
    return outputs

def bert_add_validation_outputs(model, predictions, losses):
    outputs = {}
    for pred in predictions:
        outputs[pred] = popart.AnchorReturnType("ALL")
    for loss in losses:
        outputs[loss] = popart.AnchorReturnType("ALL")
    for out in outputs.keys():
        model.builder.addOutputTensor(out)
    return outputs

def bert_session_options(args, model):
    engine_options = {}
    options = popart.SessionOptions()
    options.virtualGraphMode = popart.VirtualGraphMode.Manual
    options.enableFloatingPointChecks = args.floating_point_exceptions
    options.enableStochasticRounding = args.stochastic_rounding
    options.enableGroupedMatmuls = False
    options.enablePrefetchDatastreams = not args.low_latency_inference
    options.enableOutlining = not args.no_outlining
    partials_type = "half" if args.enable_half_partials else "float"
    options.partialsTypeMatMuls = partials_type
    options.convolutionOptions = {'partialsType': partials_type}
    # Increasing the outlineThreshold prevents creating subgraphs of cheap Ops
    # such as add or reshapeInplace.
    # Instead only reusing ops with a highSubgraphValue such as matmul or normalisation.
    options.outlineThreshold = 10.0
    if args.execution_mode == "PIPELINE":
        options.enablePipelining = True
        options.autoRecomputation = popart.RecomputationType.Pipeline
    elif args.execution_mode == "PHASED":
        options.virtualGraphMode = popart.VirtualGraphMode.ExecutionPhases
        options.enableOutliningCopyCostPruning = False
        options.outlineThreshold = -np.inf
        options.executionPhaseSettings.phases = model.total_execution_phases
        options.batchSerializationSettings.factor = args.batch_serialize
        options.autoRecomputation = popart.RecomputationType.Standard
        options.explicitRecomputation = True
        options.aliasZeroCopy = True

        options.activationTensorLocationSettings.location.storage = popart.TensorStorage.OffChip

        varLocation = popart.TensorLocation()
        varLocation.storage = popart.TensorStorage.OffChip
        varLocation.loadTileSet = popart.TileSet.IO
        varLocation.storageTileSet = popart.TileSet.IO
        varLocation.replicatedTensorSharding = (popart.ReplicatedTensorSharding.On
                                                if args.replicated_weight_sharding else
                                                popart.ReplicatedTensorSharding.Off)

        options.weightTensorLocationSettings.location = varLocation
        options.optimizerStateTensorLocationSettings.location = varLocation
        options.accumulatorTensorLocationSettings.location = varLocation

        options.numIOTiles = args.num_io_tiles
        options.timeLimitScheduler = -1
        options.swapLimitScheduler = -1
        engine_options["target.syncReplicasIndependently"] = "false"

    if args.gradient_accumulation_factor > 1:
        options.enableGradientAccumulation = True
        options.accumulationFactor = args.gradient_accumulation_factor
    if args.replication_factor > 1:
        options.enableReplicatedGraphs = True
        options.replicatedGraphCount = args.replication_factor
    if args.engine_cache is not None:
        options.enableEngineCaching = True
        options.cachePath = args.engine_cache
    if args.gc_profile:
        options.enableEngineCaching = False
        options.reportOptions = {
            "showVarStorage": "true",
            "showPerIpuMemoryUsage": "true",
            "showExecutionSteps": "true"
        }
    options.instrumentWithHardwareCycleCounter = args.report_hw_cycle_count
    options.disableGradAccumulationTensorStreams = True
    if args.max_copy_merge_size == -1:
        logger.debug("No copy merge size limit applied")
    else:
        logger.warning(
            f"Copy merge size limit set to {args.max_copy_merge_size}")
        engine_options["opt.maxCopyMergeSize"] = str(args.max_copy_merge_size)

    # Adding {"fullyConnectedPass", "TRAINING_BWD"} to some matmuls causes large
    # transposes before operations.
    if args.disable_fully_connected_pass:
        if args.task == "SQUAD" and args.sequence_length == 384:
            logger.warning(
                "Fully connected pass has been disabled. This may cause SQuAD 384 12-layer to go OOM.")
        options.enableFullyConnectedPass = False

    if args.inference and args.engine_cache is not None and not args.variable_weights_inference:
        logger.warning("Using engine cache with constant weights. Checkpoint weights will be ignored. "
                       "Use the `--variable-weights-inference` flag if checkpoint weights should be used.")

    if args.variable_weights_inference:
        options.constantWeights = False

    if args.group_host_syncs:
        options.groupHostSync = True

    if args.internal_exchange_optimisation_target is not None:
        engine_options["opt.internalExchangeOptimisationTarget"] = str(args.internal_exchange_optimisation_target)

    options.engineOptions = engine_options

    # Set synthetic data mode (if active)
    if args.synthetic_data:
        if args.synthetic_data_initializer == "zeros":
            options.syntheticDataMode = popart.SyntheticDataMode.Zeros
        else:
            options.syntheticDataMode = popart.SyntheticDataMode.RandomNormal
        logger.info(
            f"Running with Synthetic Data Type '{options.syntheticDataMode}'")
    return options

def bert_session_patterns(args):
    patterns = popart.Patterns()
    if args.task != "SQUAD":
        patterns.enablePattern("DisableAttnDropoutBwdPattern", False)

    if args.execution_mode == ExecutionMode.PHASED:
        patterns.enablePattern("TiedGatherPattern", False)
    return patterns

def calc_required_ipus(args, model):
    if args.execution_mode == "PHASED":
        num_ipus = 2
    else:
        num_ipus = model.total_ipus
    num_ipus *= args.replication_factor
    request_ipus = pow(2, math.ceil(math.log2(num_ipus)))
    logger.info(f"Need {num_ipus} IPUs. Requesting {request_ipus}")
    return request_ipus, num_ipus

def compile_graph_checked(args, session):
    try:
        start_time = time.time()
        session.prepareDevice()
        end_time = time.time()
        logger.info(f"Compiled. Duration {end_time - start_time} seconds")
    except popart.OutOfMemoryException as e:
        utils.fetch_reports(args, session=session, exception=e)
        raise e

def bert_training_session(model, args, feed, losses, device,
                          optimizer_factory):
    options = bert_session_options(args, model)

    patterns = bert_session_patterns(args)

    def final_loss(losses):
        if model.config.execution_mode == ExecutionMode.PHASED:
            with model.scope_provider(model.builder, model.final_loss_scope):
                reduced_losses = []
                for loss in losses:
                    reduced_losses.append(
                        model.builder.aiOnnx.reducesum([loss], [0, 1], 0, f"Reduce{loss}"))
                loss_sum = model.builder.aiOnnx.sum(reduced_losses, "FinalLoss")
        else:
            with model.final_loss_scope:
                reduced_losses = []
                for loss in losses:
                    reduced_losses.append(
                        model.builder.aiOnnx.reducesum([loss], [0, 1], 0, f"Reduce{loss}"))
                loss_sum = model.builder.aiOnnx.sum(reduced_losses, "FinalLoss")
        return loss_sum

    loss = final_loss(losses)
    proto = model.builder.getModelProto()

    optimizer = optimizer_factory.create()

    logger.info("Creating Session")
    session = popart.TrainingSession(fnModel=proto,
                                     loss=loss,
                                     deviceInfo=device,
                                     optimizer=optimizer,
                                     dataFlow=feed,
                                     patterns=patterns,
                                     userOptions=options)

    logger.info("Compiling Training Graph")
    compile_graph_checked(args, session)

    utils.fetch_reports(args, session=session)

    session.weightsFromHost()
    session.setRandomSeed(args.seed)

    anchors = session.initAnchorArrays()

    return session, anchors

def bert_inference_session(model, args, feed, device):
    options = bert_session_options(args, model)

    patterns = bert_session_patterns(args)

    proto = model.builder.getModelProto()

    logger.info("Creating Session")
    session = popart.InferenceSession(fnModel=proto,
                                      deviceInfo=device,
                                      dataFlow=feed,
                                      patterns=patterns,
                                      userOptions=options)

    logger.info("Compiling Inference Graph")
    compile_graph_checked(args, session)

    utils.fetch_reports(args, session=session)

    session.weightsFromHost()
    session.setRandomSeed(args.seed)

    anchors = session.initAnchorArrays()

    return session, anchors

def bert_writer(args):
    log_name = f"{os.path.basename(args.checkpoint_dir)}."\
               f"{datetime.datetime.now().isoformat()}"
    log_dir = os.path.join(
        args.log_dir, log_name)
    writer = SummaryWriter(log_dir=log_dir)
    return writer

def get_bert_dataset(model, args, inputs, embedding_dict=None, positional_dict=None, merge_both_embeddings=False):
    config = model.config
    shapeOf = model.builder.getTensorShape
    # The inputs after the first three (ind, pos, seg) are always lists
    inputs = reduce(chain, inputs[3:], inputs[:3])
    tensor_shapes = [(tensorId, shapeOf(tensorId)) for tensorId in inputs]

    if config.task == "PRETRAINING":
        return get_pretraining_dataset(
            tensor_shapes,
            input_files=args.input_files,
            sequence_length=config.sequence_length,
            mask_tokens=config.mask_tokens,
            vocab_length=config.vocab_length,
            batch_size=config.batch_size,
            batches_per_step=args.batches_per_step,
            accumulation_factor=args.gradient_accumulation_factor,
            replication_factor=args.replication_factor,
            duplication_factor=args.duplication_factor,
            shuffle=args.shuffle,
            generated_data=args.generated_data or args.synthetic_data,
            epochs_to_cache=args.epochs_to_cache,
            start_data_at_epoch=args.continue_training_from_epoch)

    if config.task == "SQUAD":
        ds = get_squad_dataset(
            tensor_shapes,
            input_file=args.input_files[0],
            output_dir=args.squad_results_dir,
            sequence_length=config.sequence_length,
            vocab_file=args.vocab_file,
            vocab_length=config.vocab_length,
            batch_size=config.batch_size,
            batches_per_step=args.batches_per_step,
            embedding_dict=embedding_dict,
            positional_dict=positional_dict,
            merge_both_embeddings=merge_both_embeddings,
            accumulation_factor=args.gradient_accumulation_factor,
            replication_factor=args.replication_factor,
            shuffle=args.shuffle,
            is_training=not args.inference,
            overwrite_cache=args.overwrite_cache,
            no_drop_remainder=args.no_drop_remainder,
            evaluate_script=args.squad_evaluate_script,
            generated_data=args.generated_data or args.synthetic_data,
            do_lower_case=args.do_lower_case,
            max_pipeline_stage=model.total_pipeline_stages if args.execution_mode == "PIPELINE" else 1,
            seed=args.seed,
            mpi_size=args.mpi_size,
            mpi_rank=args.mpi_rank,
            is_distributed= args.mpi_size > 1)
        return ds

def bert_step_loss(losses, anchors, num_unmasked, padding_masks):
    master_mask = num_unmasked != 0
    for loss, mask in zip(losses, padding_masks):
        anchors[loss][np.logical_not(mask)] = 0
    combined_loss = reduce(np.add, map(lambda loss: anchors[loss], losses))
    # Mask both num_losses and combined_loss to remove entries where all labels are ignored
    num_unmasked = num_unmasked[master_mask]
    combined_loss = combined_loss[master_mask]
    # Calculate mean loss for each token
    combined_loss /= num_unmasked
    # Calculate mean loss for step
    step_loss = np.mean(combined_loss)
    return step_loss

def bert_step_accuracy(labels, anchors, predictions, num_unmasked, padding_masks):
    total_correct = 0
    for pred, label, mask in zip(map(lambda p: anchors[p], predictions),
                                 labels,
                                 padding_masks):
        equal = pred.reshape(label.shape) == label
        total_correct += np.sum(equal[mask])
    total_attempted = np.sum(num_unmasked)
    step_accuracy = total_correct / total_attempted
    return step_accuracy

def bert_output_stats(labels, anchors, losses, predictions, ignore_index=None):
    if ignore_index is not None:
        padding_masks = [label != ignore_index for label in labels]
    else:
        padding_masks = [np.ones(label.shape, np.bool) for label in labels]
    num_unmasked = np.sum(np.array(padding_masks, dtype=np.int8), axis=0)

    # In the case of inference w/ labels, we don't care about the loss, so it can be ignored
    if losses[0] is None:
        step_loss = None
    else:
        step_loss = bert_step_loss(losses, anchors, num_unmasked, padding_masks)

    step_accuracy = bert_step_accuracy(labels, anchors, predictions, num_unmasked, padding_masks)
    return step_loss, step_accuracy

def bert_pretraining_stats(labels, anchors, losses, predictions):
    if losses is None:
        losses = [None, None]

    # For pretraining inference with perplexity, we'll only have a single loss
    if len(losses) == 1:
        losses.append(None)

    mlm_loss, mlm_acc = bert_output_stats(
        [labels[0]], anchors, [losses[0]], [predictions[0]], 0)
    nsp_loss, nsp_acc = bert_output_stats(
        [labels[1]], anchors, [losses[1]], [predictions[1]], 2)

    return [mlm_loss, nsp_loss], [mlm_acc, nsp_acc]

def save_model_and_stats(args, session, writer, step, epoch=None, step_in_filename=False):
    if not args.no_model_save:
        save_file = "model"
        if epoch is not None:
            save_file += f"_{epoch}"
        if step_in_filename:
            save_file += f":{step}"
        save_file += '.onnx'
        save_path = os.path.join(args.checkpoint_dir, save_file)
        logger.info(f"Saving model to: {save_path}")
        session.modelToHost(save_path)
        utils.save_model_statistics(save_path, writer, step)

class Iteration:
    def __init__(self, args, batches_per_step, steps_per_epoch, writer, recording_steps=None):
        self.start_epoch = args.continue_training_from_epoch
        self.count = self.start_epoch * steps_per_epoch
        self.epoch = 0
        self.epochs = args.epochs
        self.epochs_per_save = args.epochs_per_save
        self.steps_per_log = args.steps_per_log
        self.samples_per_step = batches_per_step * \
            args.gradient_accumulation_factor * args.replication_factor * args.batch_size
        self.steps_per_epoch = steps_per_epoch
        self.total_steps = self.steps_per_epoch * self.epochs
        self.writer = writer
        self.task = args.task
        self.calculate_perplexity = args.inference_lm_perplexity
        # This should get overridden but will ensure we can always write a scalar to TB.
        self.learning_rate = 0
        if recording_steps is None:
            recording_steps = self.steps_per_epoch
        self.durations = deque(maxlen=recording_steps)
        self.cycles = deque(maxlen=recording_steps)
        if self.task == "PRETRAINING":
            self.mlm_losses = deque(maxlen=recording_steps)
            self.nsp_losses = deque(maxlen=recording_steps)
            self.mlm_accuracies = deque(maxlen=recording_steps)
            self.nsp_accuracies = deque(maxlen=recording_steps)
            self.stats_fn = bert_pretraining_stats
        else:
            self.losses = deque(maxlen=recording_steps)
            self.accuracies = deque(maxlen=recording_steps)
            self.stats_fn = bert_output_stats

    def add_stats(self, duration, hw_cycles, *args):
        self.durations.append(duration)
        if hw_cycles:
            self.cycles.append(hw_cycles)
        loss, accuracy = self.stats_fn(*args)
        self.writer.add_scalar("defaultLearningRate",
                               self.learning_rate,
                               self.count)
        if self.task == "PRETRAINING":
            self.mlm_losses.append(loss[0])
            self.nsp_losses.append(loss[1])
            self.mlm_accuracies.append(accuracy[0])
            self.nsp_accuracies.append(accuracy[1])
            self.writer.add_scalar("loss/MLM",
                                   np.average(self.mlm_losses),
                                   self.count)
            self.writer.add_scalar("loss/NSP",
                                   np.average(self.nsp_losses),
                                   self.count)
            self.writer.add_scalar("accuracy/MLM",
                                   np.average(self.mlm_accuracies),
                                   self.count)
            self.writer.add_scalar("accuracy/NSP",
                                   np.average(self.nsp_accuracies),
                                   self.count)
        else:
            self.losses.append(loss)
            self.accuracies.append(accuracy)
            self.writer.add_scalar("loss",
                                   np.average(self.losses),
                                   self.count)
            self.writer.add_scalar("accuracy",
                                   np.average(self.accuracies),
                                   self.count)

    def add_inference_stats(self, duration, hw_cycles, labels_data, anchors, predictions, losses):
        self.durations.append(duration)
        if hw_cycles:
            self.cycles.append(hw_cycles)

        if labels_data:

            loss, accuracy = self.stats_fn(labels_data, anchors, losses, predictions)

            if self.task == "PRETRAINING":
                self.mlm_accuracies.append(accuracy[0])
                self.nsp_accuracies.append(accuracy[1])

                if self.calculate_perplexity:
                    self.mlm_losses.append(loss[0])
            else:
                self.accuracies.append(accuracy)

    @property
    def throughput(self):
        return np.divide(self.samples_per_step, self.durations)

    def report_stats(self):
        avg = np.average
        status_string = \
            f"Iteration: {self.count:6} " \
            f"Epoch: {self.count/self.steps_per_epoch:6.2f}/{self.epochs} "
        if self.task == "PRETRAINING":
            status_string += \
                f"Loss (MLM NSP): {avg(self.mlm_losses):5.3f} {avg(self.nsp_losses):5.3f} " \
                f"Accuracy (MLM NSP): {avg(self.mlm_accuracies):5.3f} {avg(self.nsp_accuracies):5.3f} "
        else:
            status_string += \
                f"Loss: {avg(self.losses):5.3f} " \
                f"Accuracy: {avg(self.accuracies):5.3f} "
        status_string += \
            f"Learning Rate: {self.learning_rate:.5f} "
        status_string += \
            f"Duration: {avg(self.durations):6.4f} s " \
            f"Throughput: {avg(self.throughput):6.1f} samples/s"
        if self.cycles:
            status_string += f" Cycles: {int(avg(self.cycles))}"
        logger.info(status_string)

    def report_inference_stats(self, mean_latency, min_latency, max_latency, hw_cycles):
        avg = np.average
        status_string = \
            f"Iteration: {self.count:6} " \
            f"Duration: {avg(self.durations):6.4f} s " \
            f"Throughput: {avg(self.throughput):6.1f} samples/s"

        if self.task == "PRETRAINING":
            status_string += \
                f" Accuracy (MLM NSP): {avg(self.mlm_accuracies):5.3f} {avg(self.nsp_accuracies):5.3f}"

            if self.calculate_perplexity:
                status_string += \
                    f" LM Perplexity: {np.exp(avg(self.mlm_losses)):5.3f}"

        if mean_latency is not None:
            status_string += f" Per-sample Latency: {mean_latency} {min_latency} {max_latency} seconds (mean min max)"
        if hw_cycles is not None:
            status_string += f" Cycles: {hw_cycles}"
        logger.info(status_string)

def bert_process_data(args, session, labels, data, anchors,
                      losses, predictions, iteration: Iteration,
                      optimizer_factory: ScheduledOptimizerFactory):
    labels_data = [data[label] for label in labels]
    if not np.any([np.any(label) for label in labels_data]):
        # Label may be all padding due to args.vocab_length being smaller than when the data was generated
        return

    stepio = popart.PyStepIO(data, anchors)

    start = time.time()
    session.run(stepio)
    duration = time.time() - start
    hw_cycles = session.getCycleCount() if args.report_hw_cycle_count else None

    iteration.add_stats(duration, hw_cycles, labels_data,
                        anchors, losses, predictions)

    if (iteration.count % iteration.steps_per_log) == 0:
        iteration.report_stats()

    utils.fetch_reports(args, session=session, execution=True)

    # The following will only be true if:
    #   Learning rate mode is STEP and the current total step counter is in the schedule
    #   Learning rate mode is EPOCH and the current epoch has just changed to one in the schedule
    if optimizer_factory.should_update(iteration):
        optimizer = optimizer_factory.update_and_create(iteration)
        session.updateOptimizerFromHost(optimizer)

    iteration.count += 1

def get_timing_start_anchor(start_times):
    # Return the ID of the first input that is sent from the host.
    # Order is repeateable so we can just check the time for one entry:
    return min(start_times, key=lambda k: start_times[k][-1])

def get_timing_end_anchor(end_times):
    # Return the ID of the last anchor that is returned to the host.
    # Order is repeateable so we can just check the time for one entry:
    return max(end_times, key=lambda k: end_times[k][-1])

def create_callback_stepio(data, anchors, start_times, end_times, batches_per_step):

    micro_batch_indices = defaultdict(int)

    # Input callback is called when the data is needed:
    def input_callback(id, is_prefetch: bool):
        input_time = time.perf_counter()
        start_times[id].append(input_time)

        return data[id][micro_batch_indices[id]]

    # Called after the input buffer has been consumed by the device:
    def input_complete_callback(id):
        micro_batch_indices[id] = \
            (micro_batch_indices[id] + 1) % batches_per_step
        return

    # Output callback is called when a buffer is needed for the result:
    def output_callback(id):
        return anchors[id][micro_batch_indices[id]]

    # Complete callback is called when the output buffer has
    # been filled (result is ready to be consumed by the host):
    def output_complete_callback(id):
        output_time = time.perf_counter()
        end_times[id].append(output_time)
        micro_batch_indices[id] = \
            (micro_batch_indices[id] + 1) % batches_per_step

    stepio = popart.PyStepIOCallback(input_callback,
                                     input_complete_callback,
                                     output_callback,
                                     output_complete_callback)
    return stepio

def compute_latency(args, start_times, end_times, durations):
    if start_times and args.low_latency_inference and args.task == "SQUAD":
        # Compute latency stats using time between the
        # two anchors most separated in time:
        start_id = get_timing_start_anchor(start_times)
        end_id = get_timing_end_anchor(end_times)
        rtts = list(
            map(lambda v: v[1] - v[0], zip(start_times[start_id], end_times[end_id])))
        if len(rtts) != args.batches_per_step:
            raise RuntimeError(
                "Number of timings doesn't match items in the batch. Something is wrong.")
        mean_latency = (sum(rtts)) / args.batches_per_step
        min_latency = min(rtts)
        max_latency = max(rtts)
        if (logging.getLogger().isEnabledFor(logging.DEBUG)):
            for i, v in enumerate(rtts):
                logging.debug(f"LATENCY: {i} {v}")
    else:
        mean_latency = np.average(durations)
        min_latency = min(durations)
        max_latency = max(durations)
    return mean_latency, min_latency, max_latency

def bert_process_infer_data(args, session, data, anchors,
                            logits, iteration: Iteration,
                            start_times, end_times, stepio,
                            labels=None, predictions=None, losses=None):
    if stepio is None:
        stepio = popart.PyStepIO(data, anchors)

    start = time.perf_counter()
    session.run(stepio)
    duration = time.perf_counter() - start
    hw_cycles = session.getCycleCount() if args.report_hw_cycle_count else None

    labels_data = None
    if labels is not None:
        labels_data = [data[label] for label in labels]

        if not np.any([np.any(label) for label in labels_data]):
            labels_data = None

    iteration.add_inference_stats(duration, hw_cycles, labels_data, anchors, predictions, losses)

    mean_latency, min_latency, max_latency = compute_latency(
        args, start_times, end_times, iteration.durations)

    if (iteration.count % iteration.steps_per_log) == 0:
        iteration.report_inference_stats(mean_latency, min_latency, max_latency, hw_cycles)

    utils.fetch_reports(args, session=session, execution=True)

    iteration.count += 1

    if args.task == "PRETRAINING":
        return None

    return [anchors[logit] for logit in logits]

def bert_train_loop(args, session, writer,
                    dataset, labels, predictions, losses, anchors,
                    iteration, optimizer_factory):
    losses = [loss for loss in losses]

    save_model_and_stats(args, session, writer,
                         iteration.count, iteration.epoch)

    for iteration.epoch in range(iteration.start_epoch, args.epochs):
        for data in dataset:
            bert_process_data(args, session, labels, data, anchors,
                              losses, predictions, iteration, optimizer_factory)

            if args.steps_per_save > 0 and (iteration.count % args.steps_per_save) == 0:
                save_model_and_stats(args, session, writer,
                                     iteration.count, iteration.epoch, True)

        if args.epochs_per_save > 0 and ((iteration.epoch + 1) % iteration.epochs_per_save) == 0:
            save_model_and_stats(args, session, writer,
                                 iteration.count, iteration.epoch + 1)

    save_model_and_stats(args, session, writer, iteration.count)

def enable_realtime_scheduling(args):
    if args.realtime_scheduler:
        # Use a system call to enable real-time scheduling
        # for the whole process:
        pid = os.getpid()
        logger.info(f"Enabling real-time scheduler for process: PID {pid}")
        os.system(f"sudo -n chrt --rr -p 99 {pid}")

def disable_realtime_scheduling(args):
    if args.realtime_scheduler:
        # Use a system call to reset to default scheduling
        # for the whole process:
        pid = os.getpid()
        logger.info(f"Disabling real-time scheduler for process: PID {pid}")
        os.system(f"sudo -n chrt --other -p 0 {pid}")

def bert_infer_loop(args, session,
                    dataset, inputs, logits, anchors,
                    labels, predictions, losses,
                    iteration):
    save_results = args.task == "SQUAD" and not (args.synthetic_data or args.generated_data)

    if not losses:
        losses = None

    # Create the stepio once outside of the inference loop:
    static_data = {}
    start_times = defaultdict(list)
    end_times = defaultdict(list)
    if args.low_latency_inference and args.task == "SQUAD":
        stepio = create_callback_stepio(static_data, anchors, start_times, end_times,
                                        dataset.batches_per_step)
    else:
        stepio = None

    enable_realtime_scheduling(args)

    for iteration.epoch in range(args.epochs_inference):
        for data in dataset:
            static_data.update({t: data[t] for t in inputs})
            result = bert_process_infer_data(args, session, static_data, anchors,
                                             logits, iteration,
                                             start_times, end_times, stepio,
                                             labels, predictions, losses)

            if result is not None and save_results and iteration.epoch == args.epochs_inference - 1:
                dataset.add_results(data, result)
            start_times.clear()
            end_times.clear()

    disable_realtime_scheduling(args)

    # If SQuAD save the predictions and run the evaulation script
    if save_results:
        dataset.write_predictions()

def acquire_device(args, request_ipus):
    if args.use_ipu_model:
        model_opts = {"numIPUs": request_ipus}
        if args.ipu_model_version is not None:
            model_opts["ipuVersion"] = args.ipu_model_version
        device = popart.DeviceManager().createIpuModelDevice(model_opts)
    else:
        if args.execution_mode == "PINGPONG":
            sync_pattern = popart.SyncPattern.ReplicaAndLadder
        elif args.execution_mode == "PIPELINE" and args.replication_factor <= 1:
            sync_pattern = popart.SyncPattern.SinglePipeline
        else:
            sync_pattern = popart.SyncPattern.Full
        device = popart.DeviceManager().acquireAvailableDevice(
            request_ipus,
            pattern=sync_pattern)
    if device is None:
        raise OSError("Failed to acquire IPU.")
    logger.info(f"Acquired device: {device}")
    return device

def bert_pretrained_initialisers(config, args):
    if args.synthetic_data:
        logger.info("Initialising from synthetic_data")
        return None
    if args.generated_data:
        logger.info("Initialising from generated_data")
        return None
    if args.onnx_checkpoint:
        logger.info(
            f"Initialising from ONNX checkpoint: {args.onnx_checkpoint}")
        return utils.load_initializers_from_onnx(args.onnx_checkpoint)
    if args.tf_checkpoint:
        logger.info(f"Initialising from TF checkpoint: {args.tf_checkpoint}")
        return load_initializers_from_tf(args.tf_checkpoint, True, config, args.task)
    return None

def main(args):
    set_library_seeds(args.seed)

    config = bert_config_from_args(args)

    initializers = bert_pretrained_initialisers(config, args)

    logger.info("Building Model")
    # Specifying ai.onnx opset9 for the slice syntax
    model = get_model(config,
                      mode=args.execution_mode,
                      initializers=initializers,
                      block=None)

    # If config.host_embedding is enabled, indices and positions will have the matrices instead of the index vector.
    indices, positions, segments, masks, labels = bert_add_inputs(args, model)
    logits = bert_logits_graph(model, indices, positions, segments, masks,
                               args.execution_mode)

    if args.inference:

        predictions = None
        losses = []
        if args.task == "PRETRAINING":
            # If this is a pretraining session, labels for NSP and MLM are already within the dataset,
            # so we can always calculate prediction performance
            predictions, _ = bert_infer_graph(model, logits, include_probs=False)

            if args.inference_lm_perplexity:
                losses = bert_perplexity_graph(model, logits, labels)

            outputs = bert_add_validation_outputs(model, predictions, losses)
        else:
            if args.inference_lm_perplexity:
                raise RuntimeError("Masked LM perplexity is only supported in pretraining.")

            outputs = bert_add_logit_outputs(model, logits)

        writer = None
    else:
        predictions, probs = bert_infer_graph(model, logits)
        losses = bert_loss_graph(model, probs, labels)
        outputs = bert_add_validation_outputs(model, predictions, losses)
        writer = bert_writer(args)

    embedding_dict, positional_dict = model.get_model_embeddings()

    dataset = get_bert_dataset(model,
                               args,
                               [indices, positions, segments, masks, labels],
                               embedding_dict,
                               positional_dict,
                               config.host_embedding == "MERGE")
    logger.info(f"Dataset length: {len(dataset)}")

    data_flow = popart.DataFlow(dataset.batches_per_step, outputs)

    iteration = Iteration(
        args,
        batches_per_step=dataset.batches_per_step,
        steps_per_epoch=len(dataset),
        writer=writer,
        recording_steps=args.aggregate_metrics_over_steps)

    request_ipus, required_ipus = calc_required_ipus(args, model)

    device = acquire_device(args, request_ipus)

    if args.inference:
        session, anchors = bert_inference_session(
            model, args, data_flow, device)
        logger.info("Inference Started")
        inputs = [indices, positions, segments, *masks, *labels]
        bert_infer_loop(args, session,
                        dataset, inputs, logits, anchors,
                        labels, predictions, losses, iteration)
        device.detach()
    else:
        if not args.no_training:
            optimizer_factory = ScheduledOptimizerFactory(args,
                                                          iteration,
                                                          model.tensors)

            session, anchors = bert_training_session(model,
                                                     args,
                                                     data_flow,
                                                     losses,
                                                     device,
                                                     optimizer_factory)
            logger.info("Training Started")
            bert_train_loop(args, session, writer,
                            dataset, labels, predictions, losses, anchors,
                            iteration, optimizer_factory)

            device.detach()
            logger.info("Training Finished")

    return session, iteration

def setup_logger(log_level):
    # Define a root config with a format which is simpler for console use
    logging.basicConfig(
        level=log_level,
        format='%(asctime)s %(name)s %(levelname)s %(message)s',
        datefmt='%Y-%m-%d %H:%M:%S')
    # Define a specific Handler for this file that removes the root name.
    console = logging.StreamHandler()
    console.setLevel(log_level)
    formatter = logging.Formatter('%(asctime)s %(levelname)s %(message)s',
                                  '%Y-%m-%d %H:%M:%S')
    console.setFormatter(formatter)
    logger.addHandler(console)
    logger.propagate = False

if __name__ == "__main__":

    args = utils.parse_bert_args()

    setup_logger(logging.getLevelName(args.log_level))

    logger.info("Program Start")
    logger.info("Hostname: " + socket.gethostname())
    logger.info("Command Executed: " + str(sys.argv))

    # Run the main inference/training session by default
    if args.inference or not args.no_training:
        main(args)

    # If this was a training session and validation isn't disabled; validate.
    if not args.inference and not args.no_validation and not args.no_model_save:
        logger.info("Doing Validation")
        main(utils.get_validation_args(args))

    logger.info("Program Finished")


"""
Definition of Column:
class Column:
    def __init__(self, key, value):
        self.key = key
        self.value = value
"""

from collections import OrderedDict
class MiniCassandra:

    def __init__(self):
        self.hash = {}

    """
    @param: raw_key: a string
    @param: column_key: An integer
    @param: column_value: a string
    @return: nothing
    """
    def insert(self, raw_key, column_key, column_value):
        if raw_key not in self.hash:
            self.hash[raw_key] = OrderedDict()
        self.hash[raw_key][column_key] = column_value

    """
    @param: raw_key: a string
    @param: column_start: An integer
    @param: column_end: An integer
    @return: a list of Columns
    """
    def query(self, raw_key, column_start, column_end):
        result = []
        if raw_key not in self.hash:
            return result

        self.hash[raw_key] = OrderedDict(sorted(self.hash[raw_key].items()))
        for k, v in self.hash[raw_key].items():
            if k >= column_start and k <= column_end:
                result.append(Column(k,v))
        return result


import time

from appium import webdriver
from appium.webdriver.extensions.android.gsm import GsmCallActions

class Test_Demo:
    def setup(self):
        desired_caps = {}
        desired_caps['platformName'] = 'Android'
        desired_caps['platformVersion'] = '10'
        desired_caps['deviceName'] = '84681e49'
        desired_caps['appPackage'] = 'com.tencent.wework'
        desired_caps['appActivity'] = '.launch.LaunchSplashActivity'
        # 不清空本地缓存
        desired_caps['noReset'] = 'true'
        desired_caps['dontStopAppOnReset'] = 'true'
        desired_caps['skipDeviceInitialization'] = 'true'
        desired_caps['unicodeKeyBoard'] = 'true'
        # 以模拟器形式启动，case执行结束自动回收
        # desired_caps['avd'] = 'Pixel_23_6'
        # 设置页面空闲状态为0秒
        desired_caps['settings[waitForIdleTimeout]'] = 0
        self.driver = webdriver.Remote('http://127.0.0.1:4723/wd/hub', desired_caps)
        self.driver.implicitly_wait(10)  # 隐式等待

    def teardown(self):
        self.driver.quit()

    def test_mobile_api(self):
        self.driver.make_gsm_call("13511112222", GsmCallActions.CALL)
        self.driver.send_sms("13511113333", "hello appium api")
        self.driver.set_network_connection(1)  # 设置飞行模式
        self.driver.set_network_connection(4)  # 设置回数据网络模式
        self.driver.get_screenshot_as_file("./photo/img.png")
        self.driver.start_recording_screen()
        self.driver.stop_recording_screen()
        pass


# %%%%%%%%%%%%% Machine Learning I %%%%%%%%%%%%%%%%%%
# %%%%%%%%%%%%% Authors  %%%%%%%%%%%%%%%%%%%%%%%%%%%%
# Talal Alzahrani, MD, MPH ------>Email: tsa@gwu.edu
# %%%%%%%%%%%%% Date %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
# Final version August - 01 - 2018
# %%%%%%%%%%%%% Exploratory Data Analysis  %%%%%%%%%%
# Importing the required packages
import pandas as pd
pd.set_option('display.max_columns', 100)

# Importing the data
adult = pd.read_csv('adult.csv', sep=',', index_col=0)

# printing the dataset shape
print ('-'*40 + 'Start Console' + '-'*40 + '\n')

print("Dataset No. of Rows: ", adult.shape[0])
print("Dataset No. of Columns: ", adult.shape[1])

# printing the dataset observations
print("Dataset first few rows:\n ")
print(adult.head())

print ('-'*80 + '\n')

# printing the struture of the dataset
print("Dataset info:\n ")
print(adult.info())
print ('-'*80 + '\n')

# printing the summary statistics of the dataset
print(adult.describe(include='all'))
print ('-'*80 + '\n')

# %%%%%%%%%%%%% Support vector machine  %%%%%%%%%%%%%%%%%%%%%%%%%%
# Importing the required packages
import pandas as pd
from sklearn.preprocessing import LabelEncoder
from sklearn.metrics import confusion_matrix
from sklearn.model_selection import train_test_split
from sklearn.svm import SVC
from sklearn.metrics import accuracy_score
from sklearn.metrics import classification_report
import seaborn as sns
import matplotlib.pyplot as plt
import warnings
warnings.filterwarnings("ignore")

adult = pd.read_csv('adult.csv', sep=',', index_col=0)

# encoding the features using get dummies
X_data = pd.get_dummies(adult.iloc[:, 1:])
X = X_data.values

# encoding the class with sklearn's LabelEncoder
Y_data = adult.values[:, 0]

class_le = LabelEncoder()

# fit and transform the class
y = class_le.fit_transform(Y_data)

# Spliting the dataset into train and test
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.3, random_state=100)

# creating the classifier object
clf = SVC()

# performing training
clf.fit(X_train, y_train)

# predicton on test
y_pred = clf.predict(X_test)

print("\n")
print("Classification Report: ")
print(classification_report(y_test,y_pred))
print("\n")

print("Accuracy : ", accuracy_score(y_test, y_pred) * 100)
print("\n")

# confusion matrix

conf_matrix = confusion_matrix(y_test, y_pred)
class_names = adult['MIEV'].unique()

df_cm = pd.DataFrame(conf_matrix, index=class_names, columns=class_names )

plt.figure(figsize=(5,5))

hm = sns.heatmap(df_cm, cbar=False, annot=True, square=True, fmt='d', annot_kws={'size': 20}, yticklabels=df_cm.columns, xticklabels=df_cm.columns)
hm.yaxis.set_ticklabels(hm.yaxis.get_ticklabels(), rotation=0, ha='right', fontsize=10)
hm.xaxis.set_ticklabels(hm.xaxis.get_ticklabels(), rotation=0, ha='right', fontsize=10)
plt.ylabel('True label',fontsize=20)
plt.xlabel('Predicted label',fontsize=20)
plt.tight_layout()
plt.show()


from drf_extra_fields.fields import Base64ImageField
from recipes.models import Ingredient, Recipe, RecipeIngredient, Tag
from rest_framework import serializers

from .users import UserSerializer

class TagSerializer(serializers.ModelSerializer):
    """Тэги."""
    class Meta:
        model = Tag
        fields = ('id', 'name', 'color', 'slug')

class IngredientSerializer(serializers.ModelSerializer):
    """Ингредиенты."""
    class Meta:
        model = Ingredient
        fields = ('id', 'name', 'measurement_unit')

class ShortRecipeReadSerializer(serializers.Serializer):
    """Короткая сводка рецепта."""
    name = serializers.ReadOnlyField()
    image = Base64ImageField(read_only=True)
    cooking_time = serializers.ReadOnlyField()

    class Meta:
        model = Recipe
        fields = (
            'id', 'name', 'image', 'cooking_time'
        )

class RecipeIngredientSerializer(serializers.ModelSerializer):
    """Полная сводка ингредиента."""
    id = serializers.ReadOnlyField(source='ingredient.id')
    name = serializers.ReadOnlyField(source='ingredient.name')
    measurement_unit = serializers.ReadOnlyField(
        source='ingredient.measurement_unit')

    class Meta:
        model = RecipeIngredient
        fields = ('id', 'name', 'measurement_unit', 'amount')

class ReciperCreateIngredientSerializer(serializers.ModelSerializer):
    """Добавление ингредиента."""
    id = serializers.IntegerField()

    class Meta:
        model = RecipeIngredient
        fields = ('id', 'amount')

class RecipeSerializer(serializers.ModelSerializer):
    """Полная сводка рецепта."""
    tags = TagSerializer(read_only=True, many=True)
    ingredients = RecipeIngredientSerializer(many=True, source='recipe')
    author = UserSerializer(read_only=True)
    is_favorited = serializers.BooleanField(
        default=False
    )
    is_in_shopping_cart = serializers.BooleanField(
        default=False
    )

    class Meta:
        model = Recipe
        fields = ('id', 'tags', 'author', 'ingredients',
                  'is_favorited', 'is_in_shopping_cart',
                  'name', 'image',
                  'text', 'cooking_time', 'pub_date',)

class RecipeCreateSerializer(serializers.ModelSerializer):
    """Создание/обновление рецепта."""
    ingredients = ReciperCreateIngredientSerializer(
        many=True
    )
    image = Base64ImageField()

    class Meta:
        model = Recipe
        fields = (
            'ingredients', 'tags', 'image',
            'name', 'text', 'cooking_time'
        )
        extra_kwargs = {
            'ingredients': {'required': True},
            'tags': {'required': True},
            'image': {'required': True, 'allow_blank': False},
            'name': {'required': True, 'allow_blank': False},
            'text': {'required': True, 'allow_blank': False},
            'cooking_time': {'required': True}
        }

    def validate(self, obj):
        ingredients_list = [ingredient['id'] for ingredient in obj.get(
            'ingredients')]
        if len(ingredients_list) != len(set(ingredients_list)):
            raise serializers.ValidationError(
                'Ингредиенты не должны повторяться!'
            )
        return obj

    def create(self, validated_data):
        ingredients = validated_data.pop('ingredients')
        tags = validated_data.pop('tags')
        recipe = Recipe.objects.create(**validated_data)
        RecipeIngredient.objects.bulk_create(
            RecipeIngredient(
                recipe=recipe,
                ingredient=Ingredient.objects.get(
                    id=ingredient['id']
                ),
                amount=ingredient['amount']
            )
            for ingredient in ingredients
        )
        recipe.tags.set(tags)
        return recipe

    def update(self, instance, validated_data):
        ingredients = validated_data.pop('ingredients')
        RecipeIngredient.objects.filter(
            recipe__id=instance.id
        ).delete()
        RecipeIngredient.objects.bulk_create(
            RecipeIngredient(
                recipe=instance,
                ingredient=Ingredient.objects.get(
                    id=ingredient['id']
                ),
                amount=ingredient['amount']
            )
            for ingredient in ingredients
        )
        tags = validated_data.pop('tags')
        instance.tags.set(tags)
        instance.image = validated_data.get('image', instance.image)
        instance.name = validated_data.get('name', instance.name)
        instance.text = validated_data.get('text', instance.text)
        instance.cooking_time = validated_data.get('cooking_time',
                                                   instance.cooking_time)
        instance.save()
        return instance


import argparse
import datetime
import json
import os
import pathlib
import platform
import shlex
import shutil
import subprocess
import sys
import textwrap
import requests
from urllib.parse import urlparse

def parse_arguments():
    """
    Arguments to be passed into the script
    """
    parser = argparse.ArgumentParser(
        description="Build and sign your favourite FOSS Android Apps with your own keystore!",
    )
    parser.add_argument(
        "-b",
        "--build",
        metavar="APP",
        type=str,
        nargs="*",
        help="Builds specified app",
    )
    parser.add_argument(
        "-ba",
        "--build-all",
        action="store_true",
        help="Builds all app specified in json",
    )
    parser.add_argument(
        "-l", "--list", action="store_true", help="Lists all apps in json",
    )
    parser.add_argument(
        "-a",
        "--add-app",
        type=str,
        nargs="*",
        help="Takes application specifics input form user and stores in a JSON file",
    )
    parser.add_argument(
        "-rm",
        "--remove",
        type=str,
        nargs="*",
        help="Takes application specifics input form user and stores in a JSON file",
    )
    parser.add_argument(
        "-f",
        "--from-file",
        metavar="LOCAL_FILE",
        type=str,
        help="Takes application names from text file, finds specifics from F-Droid and stores it in a new or prexisting JSON file.",
    )
    parser.add_argument(
        "-ns",
        "--no-sign",
        action="store_true",
        help="Use if you don't want to sign built apps",
    )
    parser.add_argument("--clean", action="store_false", help="Clean out directory.")
    return parser

args = parse_arguments().parse_args()

if len(sys.argv) < 2:
    parse_arguments().print_usage()
    sys.exit(1)
else:
    root_dir = os.getcwd()
    working_dir = os.path.join(root_dir, "working")
    bin_dir = os.path.join(root_dir, "bin")
    out_dir = os.path.join(root_dir, "out")
    apps_json = os.path.join(root_dir, "apps.json")
    ks_folder = os.path.join(root_dir, "keystore")
    ks_loc = os.path.join(ks_folder, "autoappbuilder.jks")
    ks_conf = os.path.join(ks_folder, "ks.conf")

    if os.path.isfile(apps_json) and os.stat(apps_json).st_size != 0:
        db = open(apps_json, "r")
        db_data = json.load(db)
        db.close()
    elif os.path.isfile(apps_json) and os.stat(apps_json).st_size == 0:
        os.remove(apps_json)
        open(apps_json, "x")
    elif os.path.isfile(apps_json) == False:
        open(apps_json, "x")

    def setup():
        """
        Check if working directory exists and create it if it doesn't
        """
        if "ANDROID_SDK_ROOT" not in os.environ:
            print("Set environment variable ANDROID_SDK_ROOT")
            sys.exit()

        if "ANDROID_HOME" not in os.environ:
            print("Set environment variable ")
            sys.exit()

        if type(shutil.which("java")) is None:
            print("Set environment variable JAVA_HOME")
            sys.exit()

        if os.path.isdir(working_dir):
            os.chmod(working_dir, 0o755)
            shutil.rmtree(working_dir)
            os.mkdir(working_dir)
        else:
            os.mkdir(working_dir)

        if os.path.isdir(bin_dir) == False:
            os.mkdir(bin_dir)

        if os.path.isdir(out_dir) == False:
            os.mkdir(out_dir)

    def check_platform():
        if platform.system() == "Windows":
            warning = textwrap.dedent(
                """\n
            ******************* WARNING ******************
            Some apps might not have gradlew.bat script,
            please consider using WSL on Windows.
            **********************************************
            """
            )
            print(warning)
            # sys.exit()

    def info_from_user(url=""):
        if url == "":
            url = input("Please enter repository URL: ").strip()
        else:
            url = url.strip()
        url_parser(url)

    def from_file(f):
        f = os.path.join(os.getcwd(), f)
        if os.path.isfile(f) == False:
            print("Specified file doesn't exist, exiting...")
            sys.exit()

        app_list = open(f, "r").readlines()
        for app in app_list:
            app = app.strip()
            url_parser(app, True)

    def url_parser(url, loop=False):
        url = urlparse(url)

        if url.netloc == "":
            name = (
                (
                    url.path.replace("/", "", 1)[::-1].replace("/", "", 1)[::-1]
                    if (url.path.replace("/", "", 1)[-1] == "/")
                    else url.path.replace("/", "", 1)
                )
                .split("/")[1]
                .lower()
            )
            repo = url.path.replace("github.com/", "")
            remote = url.path.split("/")[0].replace(".com", "")
        else:
            name = (
                (
                    url.path.replace("/", "", 1)[::-1].replace("/", "", 1)[::-1]
                    if (url.path.replace("/", "", 1)[-1] == "/")
                    else url.path.replace("/", "", 1)
                )
                .split("/")[1]
                .lower()
            )
            repo = (
                url.path.replace("/", "", 1)[::-1].replace("/", "", 1)[::-1]
                if (url.path.replace("/", "", 1)[-1] == "/")
                else url.path.replace("/", "", 1)
            )
            remote = url.netloc.replace(" ", "").replace(".com", "")

        branch = "master"

        parse_to_json(name, repo, branch)

    def parse_to_json(name, repo, branch, remote="github"):
        new_app = dict()
        attrs_list = list()
        attrs_dict = dict()

        new_app["app"] = name
        attrs_dict["repository"] = repo
        attrs_dict["branch"] = branch
        attrs_dict["remote"] = remote

        attrs_list.append(attrs_dict)
        new_app[f"{name}"] = attrs_list

        if os.path.isfile(apps_json) and os.stat(apps_json).st_size != 0:
            apps = []

            for item in db_data:
                apps.append(item["app"])

            if name not in apps:
                print(f"Adding {name.capitalize()} to db")
                db = open(apps_json, "w")
                db_data.append(new_app)
                db.write(json.dumps(db_data, indent=2))
            else:
                print(f"{name.capitalize()} already exists in db")
        else:
            with open(apps_json, "w") as db:
                final_list = [new_app]
                db.write(json.dumps(final_list, indent=2))

    def rmapp(name):
        name = name.casefold()
        apps = list()
        for item in db_data:
            apps.append(item["app"])
        new_db = db_data
        if name in apps:
            index = apps.index(name)
            db = open(apps_json, "w")
            new_db.pop(index)
            db.write(json.dumps(new_db, indent=2))
            db.close()
        else:
            print(name.capitalize() + " doesn't exist in db, exiting...")

    def get_info_from_json(name):
        name = name.casefold()
        for item in db_data:
            if item["app"] == name:
                repo = item[name][0]["repository"]
                branch = item[name][0]["branch"]
                remote = item[name][0]["remote"]
                return (repo, branch, remote)
            else:
                continue

    def get_tag(repo):
        gh_tag = f"https://api.github.com/repos/{repo}/releases/latest"
        latest_tag = requests.get(gh_tag).json()["tag_name"]
        return latest_tag

    def dl_gh_source(name, repo, latest_tag):
        tarball_name = os.path.join(working_dir, f"{name}.tar.gz")
        gh_dl = f"https://github.com/{repo}/archive/{latest_tag}.tar.gz"

        tarball = requests.get(gh_dl)
        tarball.raise_for_status()

        with open(tarball_name, "wb") as handle:
            for block in tarball.iter_content(1024):
                handle.write(block)
            handle.close()

        shutil.unpack_archive(
            filename=f"{working_dir}/{name}.tar.gz",
            extract_dir=f"{working_dir}",
            format="gztar",
        )

        os.remove(f"{working_dir}/{name}.tar.gz")
        os.chdir(root_dir)

    def clone(name, repo, branch, remote):
        if remote == "gitlab":
            clone_url = "git://gitlab.com" + repo
        elif remote == "bitbucket":
            clone_url = "git://bitbucket.com" + repo
        elif remote == "other":
            clone_url = repo

        latest_tag = get_tag(repo).replace("v", "", 1)
        app_working_dir = os.path.join(working_dir, f"{name}-{latest_tag}")
        clonecmd = f"git clone {clone_url} -b {branch} --depth 1 {app_working_dir}"
        os.system(clonecmd)

    def build(single, name, repo="", branch="", remote=""):
        setup()

        if single:
            print(f"Fetching {name} information from db...\n")
            repo, branch, remote = get_info_from_json(name)

        latest_tag = get_tag(repo)
        app_out_dir = os.path.join(out_dir, f"{name}-{latest_tag}")

        try:
            if len(os.listdir(app_out_dir)) > 0:
                print(f"Already built {name.capitalize()}, skipping...")
        except:
            print(f"Downloading {name.capitalize()} source...\n")
            if remote == "github":
                dl_gh_source(name, repo, latest_tag)
            elif remote == "gitlab":
                clone(name, repo, branch, remote)

            app_working_dir = os.path.join(working_dir, os.listdir(working_dir)[0])

            os.chdir(app_working_dir)

            print(f"Building {name}...\n")
            try:

                if platform.system() == "Windows":
                    subprocess.run(
                        shlex.split('cmd.exe /c "gradlew.bat clean build"'),
                        capture_output=False,
                    )
                else:
                    subprocess.run(
                        shlex.split("./gradlew clean build"), capture_output=False
                    )

                unsigned_apk_path = copy_build(name, latest_tag)
                if not args.no_sign:
                    signed_path = sign_build(unsigned_apk_path)
                    print(f"\nBuild successful for {name}\nFind APK at {signed_path}")
                else:
                    print(
                        f"\nBuild successful for {name}\nFind APK at {unsigned_apk_path}"
                    )
            except:
                if not single:
                    print(f"Build failed for {name}, continuing...\n")
                else:
                    print(f"Build failed for {name}, exiting...\n")

    def copy_build(name, latest_tag):
        date = datetime.date.today()
        app_out_dir = os.path.join(out_dir, f"{name}-{latest_tag}")
        unsigned_apk_name = f"{name}-{latest_tag}-unsigned"
        unsigned_apk_path = os.path.join(app_out_dir, f"{unsigned_apk_name}.apk")
        os.mkdir(app_out_dir)
        releaseapk = str(list(pathlib.Path(working_dir).rglob("*release*.apk"))[0])
        shutil.copyfile(releaseapk, unsigned_apk_path)
        return unsigned_apk_path

    def check_for_keystore():
        global ks_conf

        if not os.path.isdir(ks_folder):
            os.mkdir(ks_folder)

        os.chdir(ks_folder)

        if os.path.isfile(ks_loc) == False or os.path.isfile(ks_conf) == False:
            while True:
                default_ks_pass = input("Password must be at least 6 characters: ")
                if len(default_ks_pass) >= 6:
                    break
            default_ks_alias = "auto-app-builder"
            print("Generating keystore\n")
            subprocess.run(
                shlex.split(
                    f"keytool -genkey -v -keystore {ks_loc} -storepass {default_ks_pass} -keypass {default_ks_pass} -alias {default_ks_alias} -keyalg RSA -keysize 2048 -validity 10000"
                )
            )

            ks_conf_dict = dict()
            ks_conf_dict["ks_loc"] = ks_loc
            ks_conf_dict["ks_pass"] = default_ks_pass
            ks_conf_dict["ks_alias"] = default_ks_alias
            ks_conf = open(ks_conf, "w")
            ks_conf.write(json.dumps(ks_conf_dict, indent=2))
            ks_conf.close()

    def sign_build(unsigned_apk_path):
        build_tools_dir = os.path.join(os.environ["ANDROID_HOME"], "build-tools")
        latest_bt = os.listdir(build_tools_dir).pop()
        latest_bt_path = os.path.join(build_tools_dir, latest_bt)
        aligned_path = unsigned_apk_path.replace("unsigned", "unsigned-aligned")
        signed_path = unsigned_apk_path.replace("unsigned", "signed")
        check_for_keystore()
        ks_data = json.load(open(ks_conf, "r"))
        ks_loc = ks_data["ks_loc"]
        ks_alias = ks_data["ks_alias"]
        ks_pass = ks_data["ks_pass"]
        if os.path.isfile(unsigned_apk_path):
            if platform.system() == "Windows":
                zipalign = os.path.join(latest_bt_path, "zipalign.exe")
                apk_signer = os.path.join(latest_bt_path, "apksigner.bat")
                subprocess.run(
                    shlex.split(
                        f'"{zipalign}"  -v -p 4 {unsigned_apk_path} {aligned_path}'
                    )
                )
                subprocess.run(
                    shlex.split(
                        f'"{apk_signer}" sign --ks {ks_loc} --ks-pass pass:{ks_pass} --key-pass pass:{ks_pass} --ks-key-alias {ks_alias} --out {signed_path} {aligned_path}'
                    )
                )
            else:
                subprocess.run(
                    shlex.split(
                        f"{latest_bt_path}/zipalign  -v -p 4 {unsigned_apk_path} {aligned_path}"
                    ),
                    capture_output=False,
                )
                subprocess.run(
                    shlex.split(
                        f"{latest_bt_path}/apksigner sign --ks {ks_loc} --ks-pass pass:{ks_pass} --key-pass pass:{ks_pass} --ks-key-alias {ks_alias} --out {signed_path} {aligned_path}"
                    ),
                    capture_output=True,
                )
        os.remove(unsigned_apk_path)
        os.remove(aligned_path)
        return signed_path

    def list_all():
        for item in db_data:
            name = item["app"]
            print(name)

    def build_all():

        for item in db_data:
            name = item["app"]
            repo = item[name][0]["repository"]
            branch = item[name][0]["branch"]
            remote = item[name][0]["remote"]
            build(False, name, repo, branch, remote)

    def clean():
        os.chmod(out_dir, 0o755)
        shutil.rmtree(out_dir)

    def main():
        if args.add_app:
            for item in args.add_app:
                info_from_user(item)

        if args.from_file:
            from_file(args.file)

        if args.remove:
            for item in args.remove:
                rmapp(item)

        if args.list:
            list_all()

        if args.build:
            for item in args.build:
                item = item.casefold()
                build(True, item)
        if args.clean:
            clean()

        if args.build_all:
            build_all()

    if __name__ == "__main__":
        setup()
        check_platform()
        main()


import json
from mailru_im_async_bot.event import EventType

import state as State

async def command(bot, event):
	await bot.send_text(chat_id=event.message_author['user_id'], text=f"command {event.text}")

async def run(bot, event):
	await bot.send_text(chat_id=event.message_author['user_id'], text=f"run")

async def hello(bot, event, user):
	await bot.send_text(chat_id=event.message_author['user_id'], text=f"1 ответ на {event.text}")
	response = await user.wait_response()
	await bot.send_text(chat_id=event.message_author['user_id'], text=f"2 ответ на {response.text}")
	response = await user.wait_response()
	await bot.send_text(chat_id=event.message_author['user_id'], text=f"3 ответ на {response.text}")

async def buttons_get_cb(bot, event):
	#print('button click by %s'%(user.id))
	await bot.send_text(
		chat_id=event.message_author['user_id'],
		text="hello with buttons.",
		inline_keyboard_markup="[{}]".format(json.dumps([
			{"text": "action 1", "callback_data": "call_back_id_2", "url": "http://mail.ru"},
			{"text": "action 2", "callback_data": "call_back_id_2"},
			{"text": "action 3", "callback_data": "call_back_id_3"}
		])))

async def buttons_answer_cb(bot, event, user):
	if event.data['callback_data'] == "call_back_id_2":
		await bot.answer_callback_query(
			query_id=event.data['query_id'],
			text="hey! it's a working button 2.",
			show_alert=True
		)

	elif event.data['callback_data'] == "call_back_id_3":
		await bot.answer_callback_query(
			query_id=event.data['query_id'],
			text="hey! it's a working button 3.",
			show_alert=False
		)

#Точка входа пользователя
async def handle_session_start(bot, event, user):
	await State.start_session(bot, user, event)

	return
	await bot.send_text(chat_id=event.message_author['user_id'], text="Меню")
	response = await user.wait_response()
	if response.type == EventType.CALLBACK_QUERY:
		await bot.send_text(chat_id=event.message_author['user_id'], text=f"была нажата кнопка {response.data['callback_data']}")
	else:
		await bot.send_text(chat_id=event.message_author['user_id'], text=f"2 ответ на {response.text}")


import pytest

@pytest.mark.django_db()
class TestSignupPage:
    def test_log_out(self, client, saved_supplier0):
        client.force_login(user=saved_supplier0.supplier_account)
        response = client.get('/logout')
        assert response.status_code == 302
        assert response['Location'] == '/'
        assert response.wsgi_request.user.is_authenticated is False


from django.shortcuts import render, get_object_or_404
from django.http import HttpResponse, HttpResponseRedirect
# Create your views here.
from .models import Question # also need to import Choice
from .models import Weather
# we will need a reverse URL lookup
from django.urls import reverse

# it's a good idea to pass sensible defaults to views
def index(request): # by convention the home page is called index
    # iterate over every member in our question collection
    latest_question_list = Question.objects.order_by('-pub_date')[:3] # a slice of the first three
    # output = ', '.join([q.question_text for q in latest_question_list])
    # return HttpResponse(output)
    # better way - use the 'render' shortcut
    context = {'latest_question_list': latest_question_list}
    #     original request, template, context
    return render(request, 'polls/index.html', context)

# we an use our 404 shortcut here
def detail(request, question_id=1): # we can pass a parameter as 'question_id'
    # recommend using get_object_or_404
    question = get_object_or_404(Question, pk=question_id)
    return render(request, 'polls/detail.html', {'question':question})
    # return HttpResponse("Here's detals of question {}".format(question_id) ) # we can pass parameters

def results(request, question_id=1):
    # return HttpResponse('Results for question {}'.format(question_id) )
    question = get_object_or_404(Question, pk=question_id)
    return render(request, 'polls/results.html', {'question':question})

def vote(request, question_id):
    question = get_object_or_404(Question, pk=question_id)
    try:
        selected_choice = question.choice_set.get(pk=request.POST['choice'])
    except (KeyError, Choice.DoesNotExist):
        # Redisplay the question voting form.
        return render(request, 'polls/detail.html', {
            'question': question,
            'error_message': "You didn't select a choice.",
        })
    else:
        selected_choice.votes += 1
        selected_choice.save()
        # Always return an HttpResponseRedirect after successfully dealing
        # with POST data. This prevents data from being posted twice if a
        # user hits the Back button.
        return HttpResponseRedirect(reverse('polls:results', args=(question.id,)))

def demo_form(request):
    return render(request, 'polls/demo_form.html' )

def weather(request):
    weather_list = Weather.objects.order_by('city')
    context = {'weather_list': weather_list}
    return render(request, 'polls/weather.html', context)

def weather_form(request, weather_id):
    # is it GET or POST?
    if request.method =='POST':
        # the form has been submitted, so save the changes and then show the 'weather' view
        selected_weather = get_object_or_404(Weather, pk=request.POST['id'])
        selected_weather.country = request.POST['country']
        selected_weather.city = request.POST['city']
        selected_weather.description = request.POST['description']
        selected_weather.temperature = request.POST['temperature']
        selected_weather.save()
        weather_list = Weather.objects.order_by('city')
        context = {'weather_list': weather_list}
        return render(request, 'polls/weather.html', context)
    else:
        # for form has NOT yet been submitted - just show the weather_form view
        weather = get_object_or_404(Weather, pk=weather_id)
        context = {'weather':weather}
        return render(request, 'polls/weather_form.html', context)