# Created by: Daniel Bemerguy
# 22/01/2021 at 23:01
import pandas as pd
import numpy as np
from sklearn import feature_extraction, linear_model, model_selection, preprocessing
import nltk
from nltk.corpus import stopwords
from nltk.tokenize import TweetTokenizer
import pickle
from nltk.classify import NaiveBayesClassifier
from sklearn.pipeline import Pipeline
from sklearn.feature_extraction.text import TfidfTransformer
from sklearn.linear_model import SGDClassifier
from sklearn.feature_extraction.text import CountVectorizer, TfidfVectorizer
from sklearn.metrics import classification_report
from sklearn.metrics import accuracy_score, confusion_matrix

#Import train and test dataset
train_df = pd.DataFrame(pd.read_csv('data/train.csv',sep = ','))

stop_words = stopwords.words('english') #stop words lsit
tknzr = TweetTokenizer() #tokenizer object

#Remove stop words from tweets
def remove_stop_words(tweet):
    tokenized_tweet = tknzr.tokenize(tweet)
    processed_tweet = list(word for word in tokenized_tweet if word not in stop_words)
    processed_tweet = ' '.join(processed_tweet)
    return processed_tweet

train_df['processed_tweet'] = train_df['text'].apply(lambda x:remove_stop_words(x))

#Define feature vector containing words excluding stop words and punctuation
feature_vector = []
punctuation = ['.',':',"'",'-',';','...',',','/','..','(',')']

for i in range(0,len(train_df)):
    feature_vector.extend(list(word for word in tknzr.tokenize(train_df.loc[i,'processed_tweet']) if word not in punctuation))

print(len(train_df))
document = train_df.loc[:,['processed_tweet','target']]
document.to_csv('processed_train_data.txt',index=None)

#Function to search for the words of the review in the feature_vector
def find_feature(word_list):
    feature = {}
    for x in feature_vector:
        feature[x] = x in word_list

    return(feature)

document = document.values.tolist()
#########################
#Naive Bayes classifier
# feature_sets = [(find_feature(word_list),target) for (word_list,target) in document]
#
# #train_set,test_set = model_selection.train_test_split(feature_sets,test_size = 0.0)
#
# model = NaiveBayesClassifier.train(feature_sets)
#
# # accuracy = nltk.classify.accuracy(model, test_set)
# # print('SVC Accuracy : {}%'.format(accuracy*100))
##################################
#Pipeline
X = train_df.loc[:,'processed_tweet']
Y = train_df.loc[:,'target']
X_train, X_test, y_train, y_test = model_selection.train_test_split(X,Y,test_size = 0.25)
sgd = Pipeline([('vect', CountVectorizer()),
                ('tfidf', TfidfTransformer()),
                ('clf', SGDClassifier(loss='hinge', penalty='l2',alpha=0.0001, random_state=42, tol=None)),
               ])
sgd.fit(X, Y)

y_pred = sgd.predict(X_test)

print(y_pred[0])
# print('accuracy %s' % accuracy_score(y_pred, y_test))
# print(classification_report(y_test, y_pred,target_names=['1','0']))
# save the model to disk
filename = 'finalized_model.sav'
pickle.dump(sgd, open(filename, 'wb'))
pickle.dump(feature_vector, open("feature_vector.pickle","wb"))
print('Saved model to disk')


import pickle
import streamlit as st
import pandas as pd
from PIL import Image
model_file = (r"https://raw.githubusercontent.com/JohnTan38/Telco-Customer-Churn-Prediction/blob/main/model_C_1.bin")

with open(model_file, 'rb') as f_in:
    dv, model = pickle.load(f_in)

NB = st.select_slider('', options = [100,150,200,250,300,350,400,500,600,700,750,800,900,1000,1250,1500,1750,2000,2500,3000,3500,4000,4500,5000,5500,6000 ], value = 100)

ColorMinMax = st.markdown(''' <style> div.stSlider > div[data-baseweb = "slider"] > div[data-testid="stTickBar"] > div {
    background: rgb(1 1 1 / 0%); } </style>''', unsafe_allow_html = True)

Slider_Cursor = st.markdown(''' <style> div.stSlider > div[data-baseweb="slider"] > div > div > div[role="slider"]{
    background-color: rgb(14, 38, 74); box-shadow: rgb(14 38 74 / 20%) 0px 0px 0px 0.2rem;} </style>''', unsafe_allow_html = True)

Slider_Number = st.markdown(''' <style> div.stSlider > div[data-baseweb="slider"] > div > div > div > div
                                { color: rgb(14, 38, 74); } </style>''', unsafe_allow_html = True)

col = f''' <style> div.stSlider > div[data-baseweb = "slider"] > div > div {{
    background: linear-gradient(to right, rgb(1, 183, 158) 0%,
                                rgb(1, 183, 158) {NB}%,
                                rgba(151, 166, 195, 0.25) {NB}%,
                                rgba(151, 166, 195, 0.25) 100%); }} </style>'''

#ColorSlider = st.markdown(col, unsafe_allow_html = True)

def main():

	#image = Image.open('images/icone.png')
	#image2 = Image.open('images/image.png')
	#st.image(image,use_column_width=False)
	add_selectbox = st.sidebar.selectbox(
	"How would you like to predict?",
	("Online", "Batch"))
	st.sidebar.info('Input data for customer')
	#st.sidebar.image(image2)
	st.title("Predicting Telco Customer Churn")
	if add_selectbox == 'Online':

		#MonthlyRevenue= st.number_input('Monthly Revenue', min_value=0, max_value=900, value=0)
		MonthlyRevenue = st.select_slider('Monthly Revenue', options = [100,150,200,250,300,350,400,500,600,700,750,800,900,1000,1250,1500,1750,2000,2500,3000,3500,4000,4500,5000,5500,6000 ], value = 100)
		ColorSlider = st.markdown(col, unsafe_allow_html = True)
		#MonthlyMinutes= st.number_input('Monthly Minutes', min_value=0, max_value=6000, value=0)
		MonthlyMinutes= st.select_slider('Monthly Minutes', options = [100,250,500,750,1000,1250,1500,1750,2000,2500,3000,3500,4000,4500,5000,5500,6000 ], value = 100)
		TotalRecurringCharge = st.number_input('Total Recurring Charge', min_value=0, max_value=400, value=0)
		DirectorAssistedCalls = st.number_input('Director Assisted Calls', min_value=0, max_value=75, value=0)
		CustomerCareCalls = st.number_input('Customer Care Calls', min_value=0, max_value=200, value=0)
		ReceivedCalls = st.number_input('Received Calls', min_value=0, max_value=2500, value=0)
		OutboundCalls = st.number_input('Outbound Calls', min_value=0, max_value=600, value=0)
		InboundCalls = st.number_input('Inbound Calls', min_value=0, max_value=600, value=0)
		MonthsInService= st.number_input(' Months In Service', min_value=6, max_value=61, value=6)
		UniqueSubs= st.number_input(' Unique Subs', min_value=1, max_value=18, value=1)
		ActiveSubs = st.number_input('Active Subs', min_value=0, max_value=11, value=0)

		#RetentionCalls = st.number_input('Retention Calls', min_value=0, max_value=5, value=0)
		RetentionCalls = st.select_slider('Retention Calls', options = [1,2,3,4,5], value = 1)
		RetentionOffersAccepted = st.number_input('Retention Offers Accepted', min_value=0, max_value=3, value=0)
		IncomeGroup = st.number_input('Income Group', min_value=0, max_value=9, value=0)
		AdjustmentsToCreditRating = st.number_input('Adjustments To Credit Rating', min_value=0, max_value=25, value=0)
		CreditRating1 = st.selectbox(' Credit Rating', ['Highest', 'High', 'Good', 'Medium', 'Low', 'VeryLow', 'Lowest'])
		CreditRating = st.selectbox(' Credit Rating', [1, 2, 3, 4, 5, 6, 7])
		PrizmCode1= st.selectbox(' Customer Residence', ['Town', 'Sunurban', 'Rural', 'Other'])
		PrizmCode = st.selectbox(' Customer Residence', [1, 2, 3, 4])
		Occupation1 = st.selectbox(' Customer Occupation', ['Clerical', 'Crafts', 'Homemaker', 'Other', 'Professional', 'Retired', 'Self', 'Student'])
		Occupation = st.selectbox(' Customer Occupation', [1, 2, 3, 4, 5, 6, 7, 8])

		output= ""
		output_prob = ""
		input_dict={

				"MonthlyRevenue": MonthlyRevenue,
				"MonthlyMinutes": MonthlyMinutes,
				"TotalRecurringCharge": TotalRecurringCharge,
				"DirectorAssistedCalls": DirectorAssistedCalls,
				"CustomerCareCalls": CustomerCareCalls,
				"ReceivedCalls": ReceivedCalls,
				"OutboundCalls": OutboundCalls,
				"InboundCalls": InboundCalls,
                                "MonthsInService":MonthsInService,
                                "UniqueSubs": UniqueSubs,
                                "ActiveSubs": ActiveSubs,
				"RetentionCalls": RetentionCalls,
				"RetentionOffersAccepted": RetentionOffersAccepted,
                                "IncomeGroup": IncomeGroup,
				"AdjustmentsToCreditRating": AdjustmentsToCreditRating,
				"CreditRating": CreditRating,
                                "PrizmCode": PrizmCode,
				"Occupation": Occupation

			}

		if st.button("Predict"):
			X = dv.transform([input_dict])
			y_pred = model.predict_proba(X)[0, 1]
			churn = y_pred >= 0.5
			output_prob = float(y_pred)
			output = bool(churn)
		st.success('Churn: {0}, Risk Score: {1}'.format(output, output_prob))
	if add_selectbox == 'Batch':
		file_upload = st.file_uploader("Upload csv file for predictions", type=["csv"])
		if file_upload is not None:
			data = pd.read_csv(file_upload)
			X = dv.transform([data])
			y_pred = model.predict_proba(X)[0, 1]
			churn = y_pred >= 0.5
			churn = bool(churn)
			st.write(churn)

if __name__ == '__main__':
	main()


import os
import glob

s1_path = "/Volumes/SAMSUNG/NEWtables/S1/"

# Defines active_map
exec(open(os.path.join(s1_path, "active_map.py")).read())
active_map_2 = {i: ( -215 + 10.* ((i% 1936)//44),
                     -215 + 10 * ((i% 1936)% 44),
                     -300 + 10 * ( i//1936)    )  for i in range(118096)}

reverse_active_map = {tuple(map(int, val)): key for key, val in active_map.items()}

def create_active_mapping(active_dict, output_filename):
    with open(output_filename, "w") as file:
        file.write("PointID X Y Z\n")
        for point_id, (x, y, z) in sorted(active_dict.items()):
            file.write(f"{point_id} {x} {y} {z}\n")

def split_table_columns_v0(input_filename, output_filename):
    with open(output_filename, "w") as file:
        file.write("PointID SensorID Prob\n")
        for i, line in enumerate(open(input_filename)):
            if not i: continue
            point_id, *probs = line.rstrip().split()
            lines = [f"{point_id} {pmt_no} {prob}" for pmt_no, prob in enumerate(probs)]
            file.write("\n".join(lines) + "\n")

def split_table_columns_v1(input_filename, output_filename):
    with open(output_filename, "w") as file:
        file.write("PointID SensorID Prob\n")
        for i, line in enumerate(open(input_filename)):
            if not i: continue
            index, x, y, z, *probs = line.rstrip().split()
            point_id = reverse_active_map[tuple(map(int, (x, y, z)))]
            lines = [f"{point_id} {pmt_no} {prob}" for pmt_no, prob in enumerate(probs)]
            file.write("\n".join(lines) + "\n")

def merge_minitables(input_filenames, output_filename):
    with open(output_filename, "w") as output_file:
        output_file.write("X Y Z SensorID Prob\n")
        for filename in input_filenames:
            file_data = open(filename).read()
            if file_data[-1] != "\n": file_data = file_data + "\n"
            output_file.write(file_data)

print("Creating active map...", end="", flush=True)
active_mapping = os.path.join(s1_path, "active_mapping.dat")
#create_active_mapping(active_map, active_mapping)
print("OK")

print("Creating active map...", end="", flush=True)
active_mapping = os.path.join(s1_path, "active_mapping_2.dat")
#create_active_mapping(active_map_2, active_mapping)
print("OK")

print("Reshaping S1 table...", end="", flush=True)
s1_table_in  = os.path.join(s1_path, "S1_old", "S1tableFULL.dat")
s1_table_out = os.path.join(s1_path, "S1_old", "S1table.dat")
#split_table_columns_v0(s1_table_in, s1_table_out)
print("OK")

print("Reshaping S1 table...", end="", flush=True)
s1_table_in  = os.path.join(s1_path, "S1_august2017", "S1TableAugust2017_V1.txt")
s1_table_out = os.path.join(s1_path, "S1_august2017", "S1table.dat")
#split_table_columns_v1(s1_table_in, s1_table_out)
print("OK")

print("Reshaping S1 table...", end="", flush=True)
s1_table_in  = os.path.join(s1_path, "S1_fnal", "S1table_original.dat")
s1_table_out = os.path.join(s1_path, "S1_fnal", "S1table.dat")
#split_table_columns_v0(s1_table_in, s1_table_out)
print("OK")

print("Merging S1 tables...", end="", flush=True)
s1_files_in  = glob.glob(os.path.join(s1_path, "S1_fnal", "table_X*_Y*.dat"))
s1_table_out = os.path.join(s1_path, "S1_fnal", "S1table_merged.dat")
merge_minitables(s1_files_in, s1_table_out)
print("OK")


"""
Annotator

Annotator for .nlp files. Facilitates the addition of comments
to nlp pass files to prepare for code generation clm training.
"""

import pandas as pandas
import math
import re
import os
import typing
from typing import Union, List, Tuple, Dict
from tqdm import tqdm
import logging
import warnings
import pickle
from datasets import Dataset

logger = logging.getLogger(__name__)

class Annotator:
    def __init__(self, dataset: Union[List[str], dict]):
        """
        Initialize annotator with nlp++ dataset.
        """
        self.dataset = dataset
        self.current_index = 0
        self.finished = False
        warnings.warn('utils.annotator.Annotator is deprecated.', DeprecationWarning, stacklevel=2)

class Dataset:
    def __init__(self, dataset: Union[List[str], dict], text_column_name: str="text"):
        if isinstance(dataset, list):
            self._dataset = {text_column_name: dataset}
        else:
            self._dataset = dataset

        self.text_column_name = text_column_name
        self.column_names = [text_column_name]

        warnings.warn('utils.annotator.Dataset is deprecated.', DeprecationWarning, stacklevel=2)

    def __getitem__(self, key: Union[slice, int]):
        if isinstance(key, slice):
            result = {}
            for k, v in self._dataset:
                result[k] = v[key]
        elif isinstance(key, int):
            result = {}
            for k, v in self._dataset:
                result[k] = v[key]
        else:
            raise ValueError

        return result

    def __setitem__(self, key: int, value: Union[str, dict]):
        if isinstance(value, str):
            if len(self._dataset) > 1:
                logger.warn(f"Columns in value (1) must match dataset ({len(self._dataset)}).")
                raise ValueError
            self._dataset[self.text_column_name][key]

        elif isinstance(value, dict):
            if set(value.keys()) != set(self._dataset):
                logger.warn(f"Columns in value (1) must match dataset ({len(self._dataset)}).")
                raise ValueError

        else:
            raise ValueError("Value must be type str or dict.")

    def __len__(self):
        return len(self._dataset[self.text_column_name])

    def _set_column_names(self):
        """
        Update column_names.
        """
        self.column_names = list(self._dataset.keys())

    def rename_columns(self, column_mapping: Dict[str,str]):
        """
        Rename column(s) using mapping.
        """
        for k, v in column_mapping.items():
            if k not in self._dataset:
                raise KeyError(f"Column '{k}' not in dataset.")
            if v in self.dataset:
                raise ValueError(f"Column '{v}' already in dataset.")
            self._dataset[v] = self._dataset[k]
            del self._dataset[k]

        self._set_column_names()

    def to_hf_dataset(self):
        """
        Convert dataset to datasets.Dataset
        """
        return Dataset.from_dict(self._dataset)

    def save_dataset(self, save_path: str):
        """
        pickle dataset
        """
        with open(save_path, "wb") as f:
            pickle.dump(self.__dict__, f)

    @classmethod
    def load_dataset(cls, dataset_path: str) -> Dataset:
        """
        Load saved dataset from file.
        """
        with open(save_path, "rb") as f:
            dataset = pickle.load(f)
            return cls(dataset["_dataset"], dataset["text_column_name"])


import urllib.request
import time
import re

url = "https://mlab.taik.fi/~timo/runo1.html"
response = urllib.request.urlopen(url)
poem = response.read().decode('iso-8859-1')

poem = re.sub('<[^<]+?>', '', poem)

lines = poem.split("\n")

for line in lines:
	a=0
	if len(line.split()) == 1:
		tabs = "\t"

	else:
		tabs = ""

	print(tabs*a + line)
	a+=1
	time.sleep(0.8)


#!/usr/bin/python
# -*- coding: utf-8 -*-
"""
Documentation: .py
  Classes and functions:
    N.A.
  Description:
    This class handles the menu which appears when the player hits the pause
    button inside a running game.
"""

__author__ = "Fireclaw the Fox"
__license__ = """
Simplified BSD (BSD 2-Clause) License.
See License.txt or http://opensource.org/licenses/BSD-2-Clause for more info
"""

from menus.Menu import Menu
from direct.gui.DirectGui import DirectButton
from panda3d.core import TextNode

class MenuPause(Menu):

    def __init__(self):
        Menu.__init__(self)
        self.frameMain.setTransparency(True)
        self.frameMain.frameColor = (0.25, 0, 1, 0.25)

        screenResMultiplier = 1.33

        self.btnReturn = DirectButton(
            # size of the button
            scale = (0.25, 0.25, 0.25),
            # some temp text
            text = "ABC",
            # size of the text
            text_scale = (0.5*screenResMultiplier, 0.5, 0.5),
            # set the alignment to center
            text_align = TextNode.ACenter,
            # put the text on the left side of the button
            text_pos = (0, -0.15),
            # set the text color to white
            text_fg = (1,1,1,1),
            # set the font of the text
            text_font = self.defaultFont,
            # set the buttons images
            geom = (self.defaultBtnMaps.find("**/button_ready"),
                    self.defaultBtnMaps.find("**/button_click"),
                    self.defaultBtnMaps.find("**/button_rollover"),
                    self.defaultBtnMaps.find("**/button_disabled")),
            # set no relief
            relief = 1,
            frameColor = (0,0,0,0),
            # No sink in when press
            pressEffect = False,
            # position on the window
            pos = (0, 0, .25),
            # the event which is thrown on clickSound
            command = self.btnReturn_Click,
            # sounds that should be played
            rolloverSound = None,
            clickSound = None)
        self.btnReturn.setTransparency(1)

        self.btnQuit = DirectButton(
            scale = (0.25, 0.25, 0.25),
            # some temp text
            text = "ABC",
            text_scale = (0.5*screenResMultiplier, 0.5, 0.5),
            # set the alignment to center
            text_align = TextNode.ACenter,
            # put the text on the left side of the button
            text_pos = (0, -0.15),
            # set the text color to white
            text_fg = (1,1,1,1),
            # set the font of the text
            text_font = self.defaultFont,
            # set the buttons images
            geom = (self.defaultBtnMaps.find("**/button_ready"),
                    self.defaultBtnMaps.find("**/button_click"),
                    self.defaultBtnMaps.find("**/button_rollover"),
                    self.defaultBtnMaps.find("**/button_disabled")),
            relief = 1,
            frameColor = (0,0,0,0),
            pressEffect = False,
            pos = (0, 0, -.25),
            command = self.btnQuit_Click,
            rolloverSound = None,
            clickSound = None)
        self.btnQuit.setTransparency(1)

        self.btnReturn.reparentTo(self.frameMain)
        self.btnQuit.reparentTo(self.frameMain)

        self.recalcAspectRatio()

        # set the text of all GUI elements
        self.setText()

        self.hideBase()

    def show(self):
        self.title["text"] = _("Pause")
        self.showBase()

    def hide(self):
        self.hideBase()

    def setText(self):
        self.title["text"] = _("Main Menu")
        self.btnReturn["text"] = _("Return")
        self.btnQuit["text"] = _("Quit")

    def btnReturn_Click(self):
        base.messenger.send("PauseMenu_return")

    def btnQuit_Click(self):
        base.messenger.send("PauseMenu_quit")


from easyvec import PolyLine, Vec2
from body import Body2d

class PolygonBody(Body2d):
    def __init__(self, polyline, mass, name=None, parent=None):
        center_mass = polyline.get_center_mass()
        iz = polyline.get_Iz(center_mass) * mass
        self.polyline = polyline.add_vec(-center_mass)
        super().__init__(x=0, y=0, alpha=0, vx=0, vy=0, mass=mass, iz=iz, omega=0, name=name, parent=parent)

    def plot(self, ax, **kwargs):
        transormed_polyline = self.polyline.transform(self._mat_1).add_vec(self.vec)
        xs, ys = [], []
        for v in transormed_polyline.vecs:
            xs.append(v.x)
            ys.append(v.y)
        xs.append(xs[0])
        ys.append(ys[0])
        ax.plot(xs, ys, **kwargs)
        cm = self.to_world(Vec2(0,0))
        ax.plot([cm.x], [cm.y], 'o', color='red')

def main():
    import matplotlib.pyplot as plt
    pb = PolygonBody(PolyLine([(0,0), (3,0), (4,1), (2,2)], copy_data=True), 2)
    pb2 = PolygonBody(PolyLine([(0,0), (3,0), (4,1), (2,2)], copy_data=True), 2, name='polygon')
    fig, ax = plt.subplots()
    pb.plot(ax)
    pb2.set_orient((1,0), (1,3),(-2,0))
    pb2.plot(ax)
    ax.grid(True)
    ax.set_aspect('equal', adjustable='box')
    plt.show()
    print(pb2.get_y0())
    print(pb2.get_full_names())

if __name__ == "__main__":
    main()


import requests
import json

url="https://reqres.in/api/users"

file=open('D:\\Manoj_python\\res.json','r')
json_input=file.read()
request_json=json.loads(json_input)

print(request_json)
response=requests.post(url,request_json)
print(response.content)
#validating response code
assert response.status_code ==201


import argparse
import configparser
import os
import misc_helpers

def get_config(filename):
    """
    All settings are stored in an external text file.
    """
    config = configparser.ConfigParser()
    config.read(filename)
    return config

def get_parser():
    parser = argparse.ArgumentParser()
    parser.add_argument("-c", "--config",
        required=False,
        default="options.cfg",
        help="path to program options file")
    parser.add_argument("-s", "--song",
        required=False,
        help="use this song instead of the one in the config file [don't write the extension .ly]")
    parser.add_argument("-r", "--right",
        required=False,
        help="create song for the right hand [possible values: 0,1]")
    parser.add_argument("-l", "--left",
        required=False,
        help="create song for the left hand [possible values: 0,1]")
    parser.add_argument("-i", "--infolder",
        required=False,
        help="folder containing the lilypond files")
    return parser

def get_options_from_config(config):
    true_strings = misc_helpers.get_true_strings()

    options = {}
    options["song"] = config["MAKE_OPTIONS"]["song"]
    options["play_right"] = True if config["MAKE_OPTIONS"]["play_right"] in true_strings else False
    options["play_left"] = True if config["MAKE_OPTIONS"]["play_left"] in true_strings else False
    options["in_folder"] = misc_helpers.normalize_folder_string(config["MAKE_OPTIONS"]["in_folder"])
    options["out_folder"] = misc_helpers.normalize_folder_string(config["MAKE_OPTIONS"]["out_folder"])
    options["image_width"] = int(config["SHARED_OPTIONS"]["screen_width"])
    options["image_height"] = int(config["SHARED_OPTIONS"]["screen_height"])
    options["image_resolution"] = int(config["MAKE_OPTIONS"]["image_resolution"])
    return options

def replace_options_from_command_line(options, args):
    true_strings = misc_helpers.get_true_strings()
    if args["song"]:
        options["song"] = args["song"]
    if args["left"]:
        options["play_left"] = True if args["left"] in true_strings else False
    if args["right"]:
        options["play_right"] = True if args["right"] in true_strings else False
    if args["infolder"]:
        options["in_folder"] = misc_helpers.normalize_folder_string(args["infolder"])

def create_complete_paths(options):
    hand_string = misc_helpers.get_folder_hand_string(options)
    options["output_folder_complete"] = os.path.join(options["out_folder"], options["song"], hand_string) + "/"
    options["lily_file"] = os.path.join(options["in_folder"], options["song"] + ".ly")

def get_options():
    parser = get_parser()
    parsed_arguments = vars(parser.parse_args())
    filename = parsed_arguments["config"]
    config = get_config(filename)
    options = get_options_from_config(config)

    replace_options_from_command_line(options, parsed_arguments)
    create_complete_paths(options)
    options["staff_activations"] = [options["play_right"], options["play_left"]]

    return options


#!/usr/bin/env python3

import json
import csv
from operator import itemgetter

def read_data(filename):
    data = []
    with open(filename, 'r') as f:
        for line in f:
            l = json.loads(line)
            if 'status' in l and l['status'] == 'ERROR':
                continue
            galeid = l['docid']
            title = l['metadata']['title']
            d = {'galeid': galeid, 'title': title}
            data.append(d)
    return data

def dump_json(dump_filename, data):
    with open(f'{dump_filename}.json', 'w') as f:
        json.dump(data, f)

def normalize(text):
    remove = '()"\''
    text = ''.join(c for c in text if c not in remove)
    return text

def dump_text(dump_filename, data):
    with open(f'{dump_filename}.txt', 'w') as f:
        # normalize
        triples = []
        for d in data:
            triple = (d['galeid'], d['title'], normalize(d['title']))
            triples.append(triple)
        sort_data = sorted(triples, key=itemgetter(2))
        for triple in sort_data:
            text = f'{triple[0]}\t{triple[1]}\n'
            f.write(text)

def dump_csv(dump_filename, data):
    with open(f'{dump_filename}.csv', 'w') as f:
        fieldnames = ['galeid', 'title']
        writer = csv.DictWriter(f, fieldnames=fieldnames)
        writer.writeheader()
        for row in data:
            writer.writerow(row)

def main(infile, dump_filename_without_extension):
    data = read_data(infile)
    dump_json(dump_filename_without_extension, data)
    dump_text(dump_filename_without_extension, data)
    dump_csv(dump_filename_without_extension, data)

if __name__ == '__main__':
    # these could be made into cli-parameters.
    infile = 'gale-log.json'
    dump_filename_without_extension = 'gale_list'

    main(infile, dump_filename_without_extension)


"""
@project = BaseToolAPI
@file = statistics_tools
@author = jianwei_hu@aliyun.cm
@create_time = 2020/2/19
"""

import numpy as np
import pandas as pd
import statsmodels.api as sm
from datetime import timedelta

def ar_ma(data):
    """
    AR自相关分析
    :param data: 时间序列数据，类型：list
    :return:
    {
        'x': [],
        'y': []
    }
    """
    data = np.array(data)
    data = data.astype(np.float64)
    data = pd.Series(data)
    data.index = pd.Index(sm.tsa.datetools.dates_from_range('2000m1', length=len(data)))
    ar_ma_qp70 = sm.tsa.ARMA(data, (7, 1)).fit()
    data_pred_start = pd.date_range(data.index[-1], data.index[-1] + timedelta(weeks=20), freq='1m')[0]
    data_pred_end = pd.date_range(data.index[-1], data.index[-1] + timedelta(weeks=20), freq='1m')[-1]
    data_predict = ar_ma_qp70.predict(start=data_pred_start, end=data_pred_end, dynamic=True)
    print(type(data_predict), data_predict)
    return {'x': data_predict.index.tolist(), 'y': data_predict.values.tolist()}

def outlier(data):
    """
    野点剔除
    :param data: 原始数据
    :return: 剔除野点之后的数据
    """
    data = np.array(data)
    data_mean = np.mean(data)
    data_std = np.std(data)
    data_median = np.median(data)
    data_3 = np.percentile(data, 75)
    data_1 = np.percentile(data, 25)
    iqr = data_3 - data_1
    # 用相邻的两个数的均值替换原数据
    for i in np.arange(1, len(data) - 1):
        # i = 1
        if (data[i]>(data_3 + 3 * iqr)) | (data[i] < (data_1 - 3 * iqr)):
            data[i] = (data[i - 1] + data[i + 1]) / 2
    return data.tolist()


def predict(**kwargs):
    import pandas as pd
    import tempfile
    import boto3
    import pickle
    import json
    import warnings

    warnings.filterwarnings("ignore")

    int_cols = [
        "FINANCIALGOALSDATA_SUITABILITYSCORE",
        "uc2_employee_flag",
        "uc2_employee_total_policy_count",
        "uc2_employee_policy_value",
        "uc7_employee_ratio",
        "uc8_employee_flag",
        "POLICYCOVHDRDATA_ISSUEAGE",
        "NBPAYMENTSDATA_FORMATAMOUNT",
        "NBAGENTDATA_PERCENTAGE",
        "NBANNUITYDATA_CHARGESINCURREPLACE",
        "NBANNUITYDATA_CASHWITHAPP",
        "NBPAYMENTSDATA_COMMISSIONRETAINED",
        "NBREPLACEMENTINFODATA_ESTIMATEDVALUE",
        "NBANNUITYDATA_EXPECTEDPREM",
        "TRXHDRDATA_PROCESSEDDATEdiff",
        "POLICYCOVHDRDATA_ISSUEDATEdiff",
        "NBANNUITYDATA_APPSIGNDATEdiff",
        "NBREPLACEMENTINFODATA_PAPERWORKRECEIVEDDATEdiff",
        "APPBENEFITDATA_STARTDATEdiff",
        "CASEHDRDATA_DATEISSUEDdiff",
        "CASEHDRDATA_DATERECEIVEDdiff",
        "CASEHDRDATA_EAPPDATARECEIVEDdiff",
        "CASEHDRDATA_TIMESTAMPdiff",
        "FINANCIALGOALSDATA_TIMESTAMPdiff",
        "no_employees",
        "NBANNUITYDATA_ANNUITYPOLICYOPTION_SPDA",
        "webservices",
    ]

    cat_cols = [
        "NBANNUITYDATA_APPSIGNSTATE_VALUE",
        "POLICYHDRDATA_PRODMODELID_VALUE",
        "NBANNUITYDATA_APPSTATUS_VALUE",
        "NBANNUITYDATA_APPSOURCE_VALUE",
        "APPCOVERAGESDATA_PRODMODELID_VALUE",
        "NBANNUITYDATA_OWNERPARTYTYPE_VALUE",
        "NBANNUITYDATA_ESIGNATUREIND_VALUE",
        "NBANNUITYDATA_OWNERANNUITANTFLAG_VALUE",
        "NBANNUITYDATA_ISREPLACEMENT_VALUE",
        "NBANNUITYDATA_ISGROUP_VALUE",
        "NBPAYMENTSDATA_PREMIUMTYPE_CODE",
        "NBANNUITYDATA_ANNUITYPOLICYOPTION",
        "NBANNUITYDATA_COMMOPTIONTYPE_VALUE",
        "NBANNUITYDATA_ANNUITYPOLICYTYPE",
        "NBANNUITYDATA_ADEQUATEASSETS_VALUE",
        "NBANNUITYDATA_PURPOSEOFACCOUNT_VALUE",
        "NBPAYMENTSDATA_METHOD",
        "NBANNUITYDATA_SYSTEMATICWITHDRAWAL_VALUE",
        "NBANNUITYDATA_CHECKFORMS_VALUE",
        "NBANNUITYDATA_ROLLOVER_VALUE",
        "NBANNUITYDATA_QUALIFIED_VALUE",
        "NBAGENTDATA_ISPRIMARY_VALUE",
        "CASEHDRDATA_ISGROUP_VALUE",
        "CASEHDRDATA_STATUS_VALUE",
        "CASEHDRDATA_UNDERWRITINGSTATUS_VALUE",
        "FINANCIALGOALSDATA_EXPECTEDSURRPENALTY_CODE",
        "FINANCIALGOALSDATA_REVERSEMORTGAGE_CODE",
        "CASEHDRDATA_ESIGNATUREIND_VALUE",
        "FINANCIALGOALSDATA_ADEQUATEASSETS_VALUE",
    ]

    bool_cols = [
        "emp_level_1",
        "emp_level_2",
        "emp_level_3",
        "emp_level_4",
        "emp_level_5",
        "emp_level_6",
        "emp_level_7",
        "emp_level_8",
        "emp_level_9",
        "emp_level_10",
        "emp_level_11",
        "emp_level_12",
        "emp_level_13",
        "emp_level_14",
        "emp_level_15",
        "emp_level_16",
        "emp_level_17",
        "emp_level_18",
        "emp_level_19",
        "emp_level_20",
        "emp_level_21",
        "emp_level_22",
    ]

    qt = [
        "FINANCIALGOALSDATA_SUITABILITYSCORE",
        "uc2_employee_total_policy_count",
        "uc2_employee_policy_value",
        "uc7_employee_ratio",
        "POLICYCOVHDRDATA_ISSUEAGE",
        "NBPAYMENTSDATA_FORMATAMOUNT",
        "NBAGENTDATA_PERCENTAGE",
        "NBANNUITYDATA_CHARGESINCURREPLACE",
        "NBANNUITYDATA_CASHWITHAPP",
        "NBPAYMENTSDATA_COMMISSIONRETAINED",
        "NBREPLACEMENTINFODATA_ESTIMATEDVALUE",
        "NBANNUITYDATA_EXPECTEDPREM",
        "TRXHDRDATA_PROCESSEDDATEdiff",
        "POLICYCOVHDRDATA_ISSUEDATEdiff",
        "NBANNUITYDATA_APPSIGNDATEdiff",
        "NBREPLACEMENTINFODATA_PAPERWORKRECEIVEDDATEdiff",
        "APPBENEFITDATA_STARTDATEdiff",
        "CASEHDRDATA_DATEISSUEDdiff",
        "CASEHDRDATA_DATERECEIVEDdiff",
        "CASEHDRDATA_EAPPDATARECEIVEDdiff",
        "CASEHDRDATA_TIMESTAMPdiff",
        "FINANCIALGOALSDATA_TIMESTAMPdiff",
        "NBANNUITYDATA_ANNUITYPOLICYOPTION_SPDA",
    ]

    THRESHOLD = 0.5056200992793571

    def get_bucket_and_key_from_s3_uri(uri):
        bucket, key = uri.split("/", 2)[-1].split("/", 1)
        return bucket, key

    def download_model_from_s3(bucket_name, key):
        bucket = boto3.resource("s3").Bucket(bucket_name)
        with tempfile.NamedTemporaryFile() as fp:
            bucket.download_fileobj(key, fp)
            loaded_model = pickle.load(open(fp.name, "rb"))
        return loaded_model

    def preprocess_employeeid(df, emp_col="V_SYSTEM_LOGON_ID"):
        df["V_SYSTEM_LOGON_ID"] = df["V_SYSTEM_LOGON_ID"].fillna("None")

        df["webservices"] = df[emp_col].apply(lambda x: 1 if "WebServices" in x else 0)
        df["emp_level_1"] = df[emp_col].apply(lambda x: 1 if "160frx" in x else 0)
        df["emp_level_2"] = df[emp_col].apply(lambda x: 1 if "160mrx" in x else 0)
        df["emp_level_3"] = df[emp_col].apply(lambda x: 1 if "162blx" in x else 0)
        df["emp_level_4"] = df[emp_col].apply(lambda x: 1 if "470FJX" in x else 0)
        df["emp_level_5"] = df[emp_col].apply(lambda x: 1 if "470bmx" in x else 0)
        df["emp_level_6"] = df[emp_col].apply(lambda x: 1 if "470bpx" in x else 0)
        df["emp_level_7"] = df[emp_col].apply(lambda x: 1 if "470ckx" in x else 0)
        df["emp_level_8"] = df[emp_col].apply(lambda x: 1 if "470ddx" in x else 0)
        df["emp_level_9"] = df[emp_col].apply(lambda x: 1 if "470djl" in x else 0)
        df["emp_level_10"] = df[emp_col].apply(lambda x: 1 if "470fnx" in x else 0)
        df["emp_level_11"] = df[emp_col].apply(lambda x: 1 if "470fsx" in x else 0)
        df["emp_level_12"] = df[emp_col].apply(lambda x: 1 if "470grw" in x else 0)
        df["emp_level_13"] = df[emp_col].apply(lambda x: 1 if "470kax" in x else 0)
        df["emp_level_14"] = df[emp_col].apply(lambda x: 1 if "470kse" in x else 0)
        df["emp_level_15"] = df[emp_col].apply(lambda x: 1 if "470lcr" in x else 0)
        df["emp_level_16"] = df[emp_col].apply(lambda x: 1 if "470lgk" in x else 0)
        df["emp_level_17"] = df[emp_col].apply(lambda x: 1 if "470max" in x else 0)
        df["emp_level_18"] = df[emp_col].apply(lambda x: 1 if "470sce" in x else 0)
        df["emp_level_19"] = df[emp_col].apply(lambda x: 1 if "470tas" in x else 0)
        df["emp_level_20"] = df[emp_col].apply(lambda x: 1 if "470vrd" in x else 0)
        df["emp_level_21"] = df[emp_col].apply(lambda x: 1 if "470wbm" in x else 0)
        df["emp_level_22"] = df[emp_col].apply(lambda x: 1 if "725ccy" in x else 0)

        df["no_employees"] = df[emp_col].apply(lambda x: len(x.split(",")))

        emp_level_cols = [
            "emp_level_1",
            "emp_level_2",
            "emp_level_3",
            "emp_level_4",
            "emp_level_5",
            "emp_level_6",
            "emp_level_7",
            "emp_level_8",
            "emp_level_9",
            "emp_level_10",
            "emp_level_11",
            "emp_level_12",
            "emp_level_13",
            "emp_level_14",
            "emp_level_15",
            "emp_level_16",
            "emp_level_17",
            "emp_level_18",
            "emp_level_19",
            "emp_level_20",
            "emp_level_21",
            "emp_level_22",
        ]

        df[["webservices", "no_employees"]] = df[["webservices", "no_employees"]].apply(
            pd.to_numeric, errors="coerce"
        )
        df[emp_level_cols] = df[emp_level_cols].astype("bool")
        df = df.drop(emp_col, axis=1)
        return df

    # init resources
    for artifact in kwargs.get("artifact"):
        if artifact.get("dataName") == "combined_artifacts":
            model_bucket, model_key = get_bucket_and_key_from_s3_uri(
                artifact.get("dataValue")
            )

    quant_transformer, catboost_encoder, lgb_clf = download_model_from_s3(
        model_bucket, model_key
    )
    pd_df = pd.DataFrame(kwargs.get("inputs").get("claim"))

    # process_nan's
    pd_df["uc2_employee_ratio"] = pd_df["uc2_employee_ratio"].fillna(-1)
    pd_df["uc7_employee_ratio"] = pd_df["uc7_employee_ratio"].fillna(-1)
    pd_df["uc8_employee_mean_change_days"] = pd_df[
        "uc8_employee_mean_change_days"
    ].fillna(-1)
    pd_df["uc8_employee_ratio"] = pd_df["uc8_employee_ratio"].fillna(-1)

    pd_df["NBPAYMENTSDATA_PREMIUMTYPE_CODE"] = pd_df[
        "NBPAYMENTSDATA_PREMIUMTYPE_CODE"
    ].fillna("UNK")
    pd_df["NBANNUITYDATA_PURPOSEOFACCOUNT_VALUE"] = pd_df[
        "NBANNUITYDATA_PURPOSEOFACCOUNT_VALUE"
    ].fillna("UNK")
    pd_df["NBPAYMENTSDATA_METHOD"] = pd_df["NBPAYMENTSDATA_METHOD"].fillna("UNK")

    pd_df["NBAGENTDATA_ISPRIMARY_VALUE"] = pd_df["NBAGENTDATA_ISPRIMARY_VALUE"].fillna(
        "Blank"
    )
    pd_df["FINANCIALGOALSDATA_EXPECTEDSURRPENALTY_CODE"] = pd_df[
        "FINANCIALGOALSDATA_EXPECTEDSURRPENALTY_CODE"
    ].fillna("Blank")
    pd_df["FINANCIALGOALSDATA_REVERSEMORTGAGE_CODE"] = pd_df[
        "FINANCIALGOALSDATA_REVERSEMORTGAGE_CODE"
    ].fillna("Blank")

    # entities
    TXN_NUM = pd_df.pop("TRXNUM")

    # preprocess
    pd_df = preprocess_employeeid(pd_df)
    pd_df[cat_cols] = pd_df[cat_cols].astype("category")
    pd_df[bool_cols] = pd_df[bool_cols].astype("bool")
    pd_df = pd_df[int_cols + cat_cols + bool_cols]

    # Robust scaler
    pd_df.loc[:, qt] = quant_transformer.transform(pd_df.loc[:, qt].to_numpy())

    # Catboost encoding
    pd_df_enc = catboost_encoder.transform(pd_df[cat_cols])
    pd_df = pd_df.drop(columns=cat_cols, inplace=False)
    pd_df = pd.concat([pd_df, pd_df_enc], axis=1)

    # 1->inconsistent, 0->consistent
    pd_df["predicted_disposition"] = (
        lgb_clf.predict_proba(pd_df)[:, 1] >= THRESHOLD
    ).astype(int)

    # Payload gen
    pd_df["TRXNUM"] = TXN_NUM
    pd_df = pd_df[["TRXNUM", "predicted_disposition"]]
    prediction_json = json.loads(pd_df.to_json(orient="records"))
    predicted_claim = prediction_json if prediction_json else None

    return [
        {
            "inputDataSource": "RANDN1209:0",  # temporarily present for payload schema restriction
            "entityId": "RAND1209",  # temporarily present for payload schema restriction
            "predictedResult": predicted_claim,
        }
    ]

# print(
#     predict(
#         model_name="SIU_APPN_model_v5",
#         artifact=[
#             {
#                 "dataId": "55dcc659-d0c5-42aa-b9bf-a0325a2997b9",
#                 "dataName": "combined_artifacts",
#                 "dataType": "artifact",
#                 "dataValue": "s3://siutempbucket/tariq/combined_appn_siu_c_v5.sav",
#                 "dataValueType": "str",
#             }
#         ],
#         inputs={
#             "claim": {
#                 "shortId": {0: "2214952", 1: "2214975"},
#                 "feedback_field": {0: "other", 1: "other"},
#                 "feedback_reason": {0: "Missed NIGO", 1: "Issued IGO"},
#                 "POLICYNUMBER": {0: "AAMGE00237", 1: "AAMGE00240"},
#                 "FINANCIALGOALSDATA_SUITABILITYSCORE": {0: 15.0, 1: 15.0},
#                 "NBANNUITYDATA_APPSIGNSTATE_VALUE": {0: "NJ", 1: "NJ"},
#                 "uc5_flag": {0: 0.0, 1: 0.0},
#                 "EMPLOYEE_ID": {
#                     0: "['500901F7D9E57D2FE0531100F20A6987', '298']",
#                     1: "['500901F7D9E57D2FE0531100F20A6987', '215']",
#                 },
#                 "V_SYSTEM_LOGON_ID": {
#                     0: "['WebServices', '470ckx']",
#                     1: "['WebServices', '470sce']",
#                 },
#                 "uc2_employee_flag": {0: 0.0, 1: 0.0},
#                 "uc2_employee_total_policy_count": {0: 6255.0, 1: 5616.0},
#                 "uc2_employee_withdrawl_policy_count": {0: 0.0, 1: 0.0},
#                 "uc2_employee_policy_value": {0: 0.0, 1: 0.0},
#                 "uc2_employee_policy_withdrawl": {0: 0.0, 1: 0.0},
#                 "uc2_employee_ratio": {0: None, 1: None},
#                 "uc7_employee_flag": {0: 0.0, 1: 0.0},
#                 "uc7_employee_policy_count": {0: 6255.0, 1: 5616.0},
#                 "uc7_employee_task_count": {0: 2045.0, 1: 1603.0},
#                 "uc7_employee_high_task_count": {0: 34.0, 1: 26.0},
#                 "uc7_employee_zero_task_count": {0: 4785.0, 1: 4442.0},
#                 "uc7_employee_ratio": {0: 0.0231292517006802, 1: 0.0221465076660988},
#                 "uc8_employee_flag": {0: 1.0, 1: 0.0},
#                 "uc8_employee_policy_count": {0: 68.0, 1: 64.0},
#                 "uc8_employee_change_policy_count": {0: 1.0, 1: 0.0},
#                 "uc8_employee_mean_change_days": {0: 7.0, 1: None},
#                 "uc8_employee_party_change_count": {0: 1.0, 1: 0.0},
#                 "uc8_employee_ratio": {0: 0.0147058823529411, 1: 0.0},
#                 "numerator": {0: 4, 1: 4},
#                 "denominator": {0: 4, 1: 4},
#                 "score": {0: 0.25, 1: 0.0},
#                 "predicted_disposition": {0: "inconsistent", 1: "inconsistent"},
#                 "eligible_hypothesis": {0: "4/4", 1: "4/4"},
#                 "CONTRACTDATA_WRITINGCODE": {0: "['IND0004582']", 1: "['IND0004582']"},
#                 "TRXNUM": {0: 2214952.0, 1: 2214975.0},
#                 "TRXHDRDATA_PROCESSEDDATE": {0: "2020-10-29", 1: "2020-10-29"},
#                 "TRXHDRDATA_TRXTYPEID_VALUE": {0: "Issue", 1: "Issue"},
#                 "POLICYCOVHDRDATA_ISSUEDATE": {0: "2020-10-29", 1: "2020-10-29"},
#                 "POLICYCOVHDRDATA_ISSUEAGE": {0: 72, 1: 72},
#                 "POLICYHDRDATA_PRODMODELID_VALUE": {
#                     0: "RSL - Apollo",
#                     1: "RSL - Apollo",
#                 },
#                 "NBANNUITYDATA_APPSTATUS_VALUE": {0: "New", 1: "New"},
#                 "NBANNUITYDATA_APPDATERECEIVED": {0: "2020-09-16", 1: "2020-10-20"},
#                 "NBANNUITYDATA_APPSIGNDATE": {0: "2020-09-16", 1: "2020-10-20"},
#                 "NBANNUITYDATA_APPSOURCE_VALUE": {
#                     0: "E-App Firelight",
#                     1: "E-App Firelight",
#                 },
#                 "APPCOVERAGESDATA_PRODMODELID_VALUE": {
#                     0: "RSL - Apollo",
#                     1: "RSL - Apollo",
#                 },
#                 "NBPAYMENTSDATA_PREMIUMTYPE_VALUE": {0: None, 1: None},
#                 "NBANNUITYDATA_OWNERPARTYTYPE_VALUE": {0: "Person", 1: "Person"},
#                 "NBREPLACEMENTINFODATA_PAPERWORKRECEIVEDDATE": {0: None, 1: None},
#                 "APPBENEFITDATA_STARTDATE": {0: "2020-10-29", 1: "2020-10-29"},
#                 "NBPAYMENTSDATA_FORMATAMOUNT": {0: -1.0, 1: -1.0},
#                 "NBAGENTDATA_PERCENTAGE": {0: 100.0, 1: 100.0},
#                 "NBANNUITYDATA_ESIGNATUREIND_VALUE": {0: True, 1: True},
#                 "NBANNUITYDATA_OWNERANNUITANTFLAG_VALUE": {0: True, 1: True},
#                 "NBANNUITYDATA_CHARGESINCURREPLACE": {0: 0.0, 1: 0.0},
#                 "NBANNUITYDATA_OWNERRELATION_VALUE": {0: None, 1: None},
#                 "NBANNUITYDATA_OWNERORGTYPE_VALUE": {0: None, 1: None},
#                 "NBREPLACEMENTINFODATA_PAYMENTSOURCETYPE_CODE": {0: None, 1: None},
#                 "NBANNUITYDATA_CASHWITHAPP": {0: 0.0, 1: 0.0},
#                 "NBANNUITYDATA_PAYMENTMETHOD_VALUE": {0: None, 1: None},
#                 "NBANNUITYDATA_ISREPLACEMENT_VALUE": {0: False, 1: False},
#                 "NBANNUITYDATA_ANNUITANTMARSTAT_VALUE": {0: None, 1: None},
#                 "NBANNUITYDATA_ISGROUP_VALUE": {0: True, 1: True},
#                 "NBPAYMENTSDATA_COMMISSIONRETAINED": {0: -1.0, 1: -1.0},
#                 "NBPAYMENTSDATA_PREMIUMTYPE_CODE": {0: None, 1: None},
#                 "NBPAYMENTSDATA_NETCOMMISSIONS": {0: -1.0, 1: -1.0},
#                 "NBANNUITYDATA_APPTYPE_VALUE": {0: None, 1: None},
#                 "NBANNUITYDATA_ANNUITYPOLICYOPTION": {0: "SPDA", 1: "SPDA"},
#                 "NBANNUITYDATA_COMMOPTIONTYPE_VALUE": {0: "Option A", 1: "Option A"},
#                 "NBANNUITYDATA_ANNUITYPOLICYTYPE": {
#                     0: "BaseCov_SPDAMVA",
#                     1: "BaseCov_SPDAMVA",
#                 },
#                 "NBANNUITYDATA_ADEQUATEASSETS_VALUE": {0: True, 1: True},
#                 "NBANNUITYDATA_PURPOSEOFACCOUNT_VALUE": {
#                     0: "Premium Billing",
#                     1: "Premium Billing",
#                 },
#                 "NBREPLACEMENTINFODATA_REPLCO1035_VALUE": {0: None, 1: None},
#                 "NBREPLACEMENTINFODATA_ESTIMATEDVALUE": {0: -1.0, 1: -1.0},
#                 "NBREPLACEMENTINFODATA_PARTIALORFULL_VALUE": {0: None, 1: None},
#                 "NBPAYMENTSDATA_METHOD": {0: None, 1: None},
#                 "NBANNUITYDATA_SYSTEMATICWITHDRAWAL_VALUE": {0: False, 1: False},
#                 "NBREPLACEMENTINFODATA_EXCHANGETYPE_VALUE": {0: None, 1: None},
#                 "NBANNUITYDATA_ACCOUNTTYPE_VALUE": {0: None, 1: None},
#                 "NBANNUITYDATA_CHECKFORMS_VALUE": {0: True, 1: True},
#                 "NBANNUITYDATA_ROLLOVER_VALUE": {0: False, 1: False},
#                 "NBANNUITYDATA_QUALIFIED_VALUE": {0: True, 1: False},
#                 "NBAGENTDATA_ISPRIMARY_VALUE": {0: True, 1: True},
#                 "NBREPLACEMENTINFODATA_ACCTYPE_VALUE": {0: None, 1: None},
#                 "NBPAYMENTSDATA_AMOUNT": {0: -1.0, 1: -1.0},
#                 "NBREPLACEMENTINFODATA_PREVPLANTYPE_VALUE": {0: None, 1: None},
#                 "NBANNUITYDATA_EXPECTEDPREM": {0: 28000.0, 1: 250444.32},
#                 "NBANNUITYDATA_ANNUITANTAGE": {0: 72.0, 1: 72.0},
#                 "NBANNUITYDATA_COMMISSIONSWITHHELD": {0: 0.0, 1: 0.0},
#                 "NBANNUITYDATA_EXPECTEDSURRPENALTY_VALUE": {0: "No", 1: "No"},
#                 "CASEHDRDATA_APPSIGNDATE": {0: "2020-09-16", 1: "2020-10-20"},
#                 "CASEHDRDATA_DATEISSUED": {0: "2020-10-29", 1: "2020-10-29"},
#                 "CASEHDRDATA_DATERECEIVED": {0: "2020-09-16", 1: "2020-10-20"},
#                 "CASEHDRDATA_EAPPDATARECEIVED": {0: "2020-09-16", 1: "2020-10-20"},
#                 "CASEHDRDATA_RATELOCKEFFECTIVEDATE": {0: "2020-09-16", 1: "2020-10-29"},
#                 "CASEHDRDATA_TIMESTAMP": {
#                     0: "2020-10-29 11:13:42.935540500",
#                     1: "2020-10-29 11:56:14.714622100",
#                 },
#                 "FINANCIALGOALSDATA_TIMESTAMP": {
#                     0: "2020-09-16 19:46:04.847665300",
#                     1: "2020-10-20 19:26:36.719643600",
#                 },
#                 "CASEHDRDATA_ISGROUP_VALUE": {0: True, 1: True},
#                 "FINANCIALGOALSDATA_EXPECTEDPREM": {0: 28000.0, 1: 250444.32},
#                 "CASEHDRDATA_STATUS_VALUE": {0: "Issued", 1: "New"},
#                 "CASEHDRDATA_UNDERWRITINGSTATUS_VALUE": {
#                     0: "Last Requirement",
#                     1: "Requirement Review",
#                 },
#                 "FINANCIALGOALSDATA_EXPECTEDSURRPENALTY_CODE": {0: "No", 1: "No"},
#                 "FINANCIALGOALSDATA_REVERSEMORTGAGE_CODE": {0: "No", 1: "No"},
#                 "CASEHDRDATA_ESIGNATUREIND_VALUE": {0: True, 1: True},
#                 "FINANCIALGOALSDATA_ADEQUATEASSETS_VALUE": {0: True, 1: True},
#                 "TRXHDRDATA_PROCESSEDDATEdiff": {0: -1, 1: -1},
#                 "POLICYCOVHDRDATA_ISSUEDATEdiff": {0: -1, 1: -1},
#                 "NBANNUITYDATA_APPSIGNDATEdiff": {0: 0, 1: 0},
#                 "NBREPLACEMENTINFODATA_PAPERWORKRECEIVEDDATEdiff": {0: -1, 1: -1},
#                 "APPBENEFITDATA_STARTDATEdiff": {0: -1, 1: -1},
#                 "CASEHDRDATA_APPSIGNDATEdiff": {0: 0, 1: 0},
#                 "CASEHDRDATA_DATEISSUEDdiff": {0: -1, 1: -1},
#                 "CASEHDRDATA_DATERECEIVEDdiff": {0: 0, 1: 0},
#                 "CASEHDRDATA_EAPPDATARECEIVEDdiff": {0: 0, 1: 0},
#                 "CASEHDRDATA_RATELOCKEFFECTIVEDATEdiff": {0: 0, 1: -1},
#                 "CASEHDRDATA_TIMESTAMPdiff": {0: -1, 1: -1},
#                 "FINANCIALGOALSDATA_TIMESTAMPdiff": {0: 0, 1: 0},
#                 "no_employees": {0: 1, 1: 1},
#                 "NBANNUITYDATA_APPSIGNSTATE_VALUE_AK": {0: 0, 1: 0},
#                 "NBANNUITYDATA_APPSIGNSTATE_VALUE_AL": {0: 0, 1: 0},
#                 "NBANNUITYDATA_APPSIGNSTATE_VALUE_AR": {0: 0, 1: 0},
#                 "NBANNUITYDATA_APPSIGNSTATE_VALUE_AZ": {0: 0, 1: 0},
#                 "NBANNUITYDATA_APPSIGNSTATE_VALUE_CA": {0: 0, 1: 0},
#                 "NBANNUITYDATA_APPSIGNSTATE_VALUE_CO": {0: 0, 1: 0},
#                 "NBANNUITYDATA_APPSIGNSTATE_VALUE_CT": {0: 0, 1: 0},
#                 "NBANNUITYDATA_APPSIGNSTATE_VALUE_DC": {0: 0, 1: 0},
#                 "NBANNUITYDATA_APPSIGNSTATE_VALUE_DE": {0: 0, 1: 0},
#                 "NBANNUITYDATA_APPSIGNSTATE_VALUE_FL": {0: 0, 1: 0},
#                 "NBANNUITYDATA_APPSIGNSTATE_VALUE_GA": {0: 0, 1: 0},
#                 "NBANNUITYDATA_APPSIGNSTATE_VALUE_HI": {0: 0, 1: 0},
#                 "NBANNUITYDATA_APPSIGNSTATE_VALUE_IA": {0: 0, 1: 0},
#                 "NBANNUITYDATA_APPSIGNSTATE_VALUE_ID": {0: 0, 1: 0},
#                 "NBANNUITYDATA_APPSIGNSTATE_VALUE_IL": {0: 0, 1: 0},
#                 "NBANNUITYDATA_APPSIGNSTATE_VALUE_IN": {0: 0, 1: 0},
#                 "NBANNUITYDATA_APPSIGNSTATE_VALUE_KS": {0: 0, 1: 0},
#                 "NBANNUITYDATA_APPSIGNSTATE_VALUE_KY": {0: 0, 1: 0},
#                 "NBANNUITYDATA_APPSIGNSTATE_VALUE_LA": {0: 0, 1: 0},
#                 "NBANNUITYDATA_APPSIGNSTATE_VALUE_MA": {0: 0, 1: 0},
#                 "NBANNUITYDATA_APPSIGNSTATE_VALUE_MD": {0: 0, 1: 0},
#                 "NBANNUITYDATA_APPSIGNSTATE_VALUE_ME": {0: 0, 1: 0},
#                 "NBANNUITYDATA_APPSIGNSTATE_VALUE_MI": {0: 0, 1: 0},
#                 "NBANNUITYDATA_APPSIGNSTATE_VALUE_MN": {0: 0, 1: 0},
#                 "NBANNUITYDATA_APPSIGNSTATE_VALUE_MO": {0: 0, 1: 0},
#                 "NBANNUITYDATA_APPSIGNSTATE_VALUE_MS": {0: 0, 1: 0},
#                 "NBANNUITYDATA_APPSIGNSTATE_VALUE_MT": {0: 0, 1: 0},
#                 "NBANNUITYDATA_APPSIGNSTATE_VALUE_NC": {0: 0, 1: 0},
#                 "NBANNUITYDATA_APPSIGNSTATE_VALUE_ND": {0: 0, 1: 0},
#                 "NBANNUITYDATA_APPSIGNSTATE_VALUE_NE": {0: 0, 1: 0},
#                 "NBANNUITYDATA_APPSIGNSTATE_VALUE_NH": {0: 0, 1: 0},
#                 "NBANNUITYDATA_APPSIGNSTATE_VALUE_NJ": {0: 1, 1: 1},
#                 "NBANNUITYDATA_APPSIGNSTATE_VALUE_NM": {0: 0, 1: 0},
#                 "NBANNUITYDATA_APPSIGNSTATE_VALUE_NV": {0: 0, 1: 0},
#                 "NBANNUITYDATA_APPSIGNSTATE_VALUE_OH": {0: 0, 1: 0},
#                 "NBANNUITYDATA_APPSIGNSTATE_VALUE_OK": {0: 0, 1: 0},
#                 "NBANNUITYDATA_APPSIGNSTATE_VALUE_OR": {0: 0, 1: 0},
#                 "NBANNUITYDATA_APPSIGNSTATE_VALUE_PA": {0: 0, 1: 0},
#                 "NBANNUITYDATA_APPSIGNSTATE_VALUE_PR": {0: 0, 1: 0},
#                 "NBANNUITYDATA_APPSIGNSTATE_VALUE_RI": {0: 0, 1: 0},
#                 "NBANNUITYDATA_APPSIGNSTATE_VALUE_SC": {0: 0, 1: 0},
#                 "NBANNUITYDATA_APPSIGNSTATE_VALUE_SD": {0: 0, 1: 0},
#                 "NBANNUITYDATA_APPSIGNSTATE_VALUE_TN": {0: 0, 1: 0},
#                 "NBANNUITYDATA_APPSIGNSTATE_VALUE_TX": {0: 0, 1: 0},
#                 "NBANNUITYDATA_APPSIGNSTATE_VALUE_UT": {0: 0, 1: 0},
#                 "NBANNUITYDATA_APPSIGNSTATE_VALUE_VA": {0: 0, 1: 0},
#                 "NBANNUITYDATA_APPSIGNSTATE_VALUE_VT": {0: 0, 1: 0},
#                 "NBANNUITYDATA_APPSIGNSTATE_VALUE_WA": {0: 0, 1: 0},
#                 "NBANNUITYDATA_APPSIGNSTATE_VALUE_WI": {0: 0, 1: 0},
#                 "NBANNUITYDATA_APPSIGNSTATE_VALUE_WV": {0: 0, 1: 0},
#                 "NBANNUITYDATA_APPSIGNSTATE_VALUE_WY": {0: 0, 1: 0},
#                 "predicted_disposition_consistent": {0: 0, 1: 1},
#                 "predicted_disposition_inconsistent": {0: 1, 1: 0},
#                 "TRXHDRDATA_TRXTYPEID_VALUE_Cost Basis Adjustment": {0: 0, 1: 0},
#                 "TRXHDRDATA_TRXTYPEID_VALUE_Death Claim": {0: 0, 1: 0},
#                 "TRXHDRDATA_TRXTYPEID_VALUE_Death Claim - 5YD": {0: 0, 1: 0},
#                 "TRXHDRDATA_TRXTYPEID_VALUE_Death Claim - LD": {0: 0, 1: 0},
#                 "TRXHDRDATA_TRXTYPEID_VALUE_Death Claim Disbursement": {0: 0, 1: 0},
#                 "TRXHDRDATA_TRXTYPEID_VALUE_Free Look Cancel": {0: 0, 1: 0},
#                 "TRXHDRDATA_TRXTYPEID_VALUE_Full Surrender": {0: 0, 1: 0},
#                 "TRXHDRDATA_TRXTYPEID_VALUE_Issue": {0: 1, 1: 1},
#                 "TRXHDRDATA_TRXTYPEID_VALUE_Partial Surrender": {0: 0, 1: 0},
#                 "TRXHDRDATA_TRXTYPEID_VALUE_Spousal Continuation": {0: 0, 1: 0},
#                 "TRXHDRDATA_TRXTYPEID_VALUE_Value Adjustment": {0: 0, 1: 0},
#                 "TRXHDRDATA_TRXTYPEID_VALUE_Death Claim - 10YD": {0: 0, 1: 0},
#                 "TRXHDRDATA_TRXTYPEID_VALUE_Annuitization": {0: 0, 1: 0},
#                 "TRXHDRDATA_TRXTYPEID_VALUE_Dividend Release": {0: 0, 1: 0},
#                 "TRXHDRDATA_TRXTYPEID_VALUE_Annuity Payment": {0: 0, 1: 0},
#                 "TRXHDRDATA_TRXTYPEID_VALUE_Death Claim Annuitization": {0: 0, 1: 0},
#                 "TRXHDRDATA_TRXTYPEID_VALUE_DOB Correction": {0: 0, 1: 0},
#                 "POLICYHDRDATA_PRODMODELID_VALUE_RSL - Apollo": {0: 1, 1: 1},
#                 "POLICYHDRDATA_PRODMODELID_VALUE_RSL - Apollo 93 BV": {0: 0, 1: 0},
#                 "POLICYHDRDATA_PRODMODELID_VALUE_RSL - Apollo 93 MVA": {0: 0, 1: 0},
#                 "POLICYHDRDATA_PRODMODELID_VALUE_RSL - Argus BV": {0: 0, 1: 0},
#                 "POLICYHDRDATA_PRODMODELID_VALUE_RSL - Argus MVA": {0: 0, 1: 0},
#                 "POLICYHDRDATA_PRODMODELID_VALUE_RSL - Cornerstone": {0: 0, 1: 0},
#                 "POLICYHDRDATA_PRODMODELID_VALUE_RSL - Deferral": {0: 0, 1: 0},
#                 "POLICYHDRDATA_PRODMODELID_VALUE_RSL - Elektra 5 6 7": {0: 0, 1: 0},
#                 "POLICYHDRDATA_PRODMODELID_VALUE_RSL - Eleos": {0: 0, 1: 0},
#                 "POLICYHDRDATA_PRODMODELID_VALUE_RSL - Eleos 96 BV": {0: 0, 1: 0},
#                 "POLICYHDRDATA_PRODMODELID_VALUE_RSL - Eleos 96 MVA": {0: 0, 1: 0},
#                 "POLICYHDRDATA_PRODMODELID_VALUE_RSL - FPDA": {0: 0, 1: 0},
#                 "POLICYHDRDATA_PRODMODELID_VALUE_RSL - John Alden": {0: 0, 1: 0},
#                 "POLICYHDRDATA_PRODMODELID_VALUE_RSL - Keystone": {0: 0, 1: 0},
#                 "POLICYHDRDATA_PRODMODELID_VALUE_RSL - Reliance Guarantee": {
#                     0: 0,
#                     1: 0,
#                 },
#                 "POLICYHDRDATA_PRODMODELID_VALUE_SSL - Converted": {0: 0, 1: 0},
#                 "POLICYHDRDATA_PRODMODELID_VALUE_RSL - Reliance Accumulator": {
#                     0: 0,
#                     1: 0,
#                 },
#                 "POLICYHDRDATA_PRODMODELID_VALUE_RSL - Deferral Annuitization": {
#                     0: 0,
#                     1: 0,
#                 },
#                 "NBANNUITYDATA_APPSTATUS_VALUE_New": {0: 1, 1: 1},
#                 "NBANNUITYDATA_APPSTATUS_VALUE_Submitted": {0: 0, 1: 0},
#                 "NBANNUITYDATA_APPSOURCE_VALUE_E-App": {0: 0, 1: 0},
#                 "NBANNUITYDATA_APPSOURCE_VALUE_E-App Firelight": {0: 1, 1: 1},
#                 "NBANNUITYDATA_APPSOURCE_VALUE_Mail": {0: 0, 1: 0},
#                 "APPCOVERAGESDATA_PRODMODELID_VALUE_RSL - Apollo": {0: 1, 1: 1},
#                 "APPCOVERAGESDATA_PRODMODELID_VALUE_RSL - Eleos": {0: 0, 1: 0},
#                 "APPCOVERAGESDATA_PRODMODELID_VALUE_RSL - Keystone": {0: 0, 1: 0},
#                 "APPCOVERAGESDATA_PRODMODELID_VALUE_RSL - Reliance Accumulator": {
#                     0: 0,
#                     1: 0,
#                 },
#                 "APPCOVERAGESDATA_PRODMODELID_VALUE_RSL - Reliance Guarantee": {
#                     0: 0,
#                     1: 0,
#                 },
#                 "NBPAYMENTSDATA_PREMIUMTYPE_VALUE_1035 Exchange": {0: 0, 1: 0},
#                 "NBPAYMENTSDATA_PREMIUMTYPE_VALUE_Direct Transfer": {0: 0, 1: 0},
#                 "NBPAYMENTSDATA_PREMIUMTYPE_VALUE_Indirect Rollover": {0: 0, 1: 0},
#                 "NBPAYMENTSDATA_PREMIUMTYPE_VALUE_New Money": {0: 0, 1: 0},
#                 "NBPAYMENTSDATA_PREMIUMTYPE_VALUE_Direct Rollover": {0: 0, 1: 0},
#                 "NBANNUITYDATA_OWNERPARTYTYPE_VALUE_Organization": {0: 0, 1: 0},
#                 "NBANNUITYDATA_OWNERPARTYTYPE_VALUE_Person": {0: 1, 1: 1},
#                 "NBANNUITYDATA_ESIGNATUREIND_VALUE_False": {0: 0, 1: 0},
#                 "NBANNUITYDATA_ESIGNATUREIND_VALUE_True": {0: 1, 1: 1},
#                 "NBANNUITYDATA_OWNERANNUITANTFLAG_VALUE_False": {0: 0, 1: 0},
#                 "NBANNUITYDATA_OWNERANNUITANTFLAG_VALUE_True": {0: 1, 1: 1},
#                 "NBANNUITYDATA_OWNERRELATION_VALUE_Business": {0: 0, 1: 0},
#                 "NBANNUITYDATA_OWNERRELATION_VALUE_Child": {0: 0, 1: 0},
#                 "NBANNUITYDATA_OWNERRELATION_VALUE_Employer": {0: 0, 1: 0},
#                 "NBANNUITYDATA_OWNERRELATION_VALUE_Other": {0: 0, 1: 0},
#                 "NBANNUITYDATA_OWNERRELATION_VALUE_Self": {0: 0, 1: 0},
#                 "NBANNUITYDATA_OWNERRELATION_VALUE_Spouse": {0: 0, 1: 0},
#                 "NBANNUITYDATA_OWNERRELATION_VALUE_Third Party Owner": {0: 0, 1: 0},
#                 "NBANNUITYDATA_OWNERRELATION_VALUE_Trust": {0: 0, 1: 0},
#                 "NBANNUITYDATA_OWNERRELATION_VALUE_Unknown": {0: 0, 1: 0},
#                 "NBANNUITYDATA_OWNERRELATION_VALUE_Estate": {0: 0, 1: 0},
#                 "NBANNUITYDATA_OWNERRELATION_VALUE_Trustee": {0: 0, 1: 0},
#                 "NBANNUITYDATA_OWNERRELATION_VALUE_Charity": {0: 0, 1: 0},
#                 "NBANNUITYDATA_OWNERRELATION_VALUE_Husband": {0: 0, 1: 0},
#                 "NBANNUITYDATA_OWNERRELATION_VALUE_Sister": {0: 0, 1: 0},
#                 "NBANNUITYDATA_OWNERRELATION_VALUE_Applicant": {0: 0, 1: 0},
#                 "NBANNUITYDATA_OWNERRELATION_VALUE_Sibling": {0: 0, 1: 0},
#                 "NBANNUITYDATA_OWNERRELATION_VALUE_Business Partner": {0: 0, 1: 0},
#                 "NBANNUITYDATA_OWNERRELATION_VALUE_Father": {0: 0, 1: 0},
#                 "NBANNUITYDATA_OWNERRELATION_VALUE_Legal Ward": {0: 0, 1: 0},
#                 "NBANNUITYDATA_OWNERRELATION_VALUE_Niece": {0: 0, 1: 0},
#                 "NBANNUITYDATA_OWNERRELATION_VALUE_Son": {0: 0, 1: 0},
#                 "NBANNUITYDATA_OWNERRELATION_VALUE_Testamentary Trust": {0: 0, 1: 0},
#                 "NBANNUITYDATA_OWNERRELATION_VALUE_Grandson": {0: 0, 1: 0},
#                 "NBANNUITYDATA_OWNERRELATION_VALUE_Nephew": {0: 0, 1: 0},
#                 "NBANNUITYDATA_OWNERORGTYPE_VALUE_Corporation": {0: 0, 1: 0},
#                 "NBANNUITYDATA_OWNERORGTYPE_VALUE_Custodial": {0: 0, 1: 0},
#                 "NBANNUITYDATA_OWNERORGTYPE_VALUE_Other": {0: 0, 1: 0},
#                 "NBANNUITYDATA_OWNERORGTYPE_VALUE_Trust": {0: 0, 1: 0},
#                 "NBREPLACEMENTINFODATA_PAYMENTSOURCETYPE_CODE_Replacement": {
#                     0: 0,
#                     1: 0,
#                 },
#                 "NBREPLACEMENTINFODATA_PAYMENTSOURCETYPE_CODE_Rollover": {0: 0, 1: 0},
#                 "NBREPLACEMENTINFODATA_PAYMENTSOURCETYPE_CODE_Transfer": {0: 0, 1: 0},
#                 "NBANNUITYDATA_PAYMENTMETHOD_VALUE_Check": {0: 0, 1: 0},
#                 "NBANNUITYDATA_PAYMENTMETHOD_VALUE_Wire": {0: 0, 1: 0},
#                 "NBANNUITYDATA_ISREPLACEMENT_VALUE_False": {0: 1, 1: 1},
#                 "NBANNUITYDATA_ISREPLACEMENT_VALUE_True": {0: 0, 1: 0},
#                 "NBANNUITYDATA_ANNUITANTMARSTAT_VALUE_Divorced": {0: 0, 1: 0},
#                 "NBANNUITYDATA_ANNUITANTMARSTAT_VALUE_Domestic Partnership": {
#                     0: 0,
#                     1: 0,
#                 },
#                 "NBANNUITYDATA_ANNUITANTMARSTAT_VALUE_Married": {0: 0, 1: 0},
#                 "NBANNUITYDATA_ANNUITANTMARSTAT_VALUE_Single": {0: 0, 1: 0},
#                 "NBANNUITYDATA_ANNUITANTMARSTAT_VALUE_Widowed": {0: 0, 1: 0},
#                 "NBANNUITYDATA_ISGROUP_VALUE_False": {0: 0, 1: 0},
#                 "NBANNUITYDATA_ISGROUP_VALUE_True": {0: 1, 1: 1},
#                 "NBPAYMENTSDATA_PREMIUMTYPE_CODE_1035EX": {0: 0, 1: 0},
#                 "NBPAYMENTSDATA_PREMIUMTYPE_CODE_DirTran": {0: 0, 1: 0},
#                 "NBPAYMENTSDATA_PREMIUMTYPE_CODE_IndRoll": {0: 0, 1: 0},
#                 "NBPAYMENTSDATA_PREMIUMTYPE_CODE_NewMoney": {0: 0, 1: 0},
#                 "NBPAYMENTSDATA_PREMIUMTYPE_CODE_DirRoll": {0: 0, 1: 0},
#                 "NBANNUITYDATA_APPTYPE_VALUE_Full Application": {0: 0, 1: 0},
#                 "NBANNUITYDATA_APPTYPE_VALUE_LITE App": {0: 0, 1: 0},
#                 "NBANNUITYDATA_ANNUITYPOLICYOPTION_SPDA": {0: 1, 1: 1},
#                 "NBANNUITYDATA_ANNUITYPOLICYOPTION_SPDA_5Yr": {0: 0, 1: 0},
#                 "NBANNUITYDATA_ANNUITYPOLICYOPTION_SPDA_7Yr": {0: 0, 1: 0},
#                 "NBANNUITYDATA_ANNUITYPOLICYOPTION_SPDA_10Yr": {0: 0, 1: 0},
#                 "NBANNUITYDATA_ANNUITYPOLICYOPTION_SPDA_6Yr": {0: 0, 1: 0},
#                 "NBANNUITYDATA_COMMOPTIONTYPE_VALUE_Option A": {0: 1, 1: 1},
#                 "NBANNUITYDATA_COMMOPTIONTYPE_VALUE_Option B": {0: 0, 1: 0},
#                 "NBANNUITYDATA_COMMOPTIONTYPE_VALUE_Option C": {0: 0, 1: 0},
#                 "NBANNUITYDATA_ANNUITYPOLICYTYPE_BaseCov_SPDA": {0: 0, 1: 0},
#                 "NBANNUITYDATA_ANNUITYPOLICYTYPE_BaseCov_SPDAMVA": {0: 1, 1: 1},
#                 "NBANNUITYDATA_ANNUITYPOLICYTYPE_BaseCov_SPEIDA": {0: 0, 1: 0},
#                 "NBANNUITYDATA_ADEQUATEASSETS_VALUE_False": {0: 0, 1: 0},
#                 "NBANNUITYDATA_ADEQUATEASSETS_VALUE_True": {0: 1, 1: 1},
#                 "NBANNUITYDATA_PURPOSEOFACCOUNT_VALUE_Disbursements": {0: 0, 1: 0},
#                 "NBANNUITYDATA_PURPOSEOFACCOUNT_VALUE_Premium Billing": {0: 1, 1: 1},
#                 "NBREPLACEMENTINFODATA_REPLCO1035_VALUE_False": {0: 0, 1: 0},
#                 "NBREPLACEMENTINFODATA_REPLCO1035_VALUE_True": {0: 0, 1: 0},
#                 "NBREPLACEMENTINFODATA_PARTIALORFULL_VALUE_Full": {0: 0, 1: 0},
#                 "NBREPLACEMENTINFODATA_PARTIALORFULL_VALUE_Partial": {0: 0, 1: 0},
#                 "NBPAYMENTSDATA_METHOD_CLH": {0: 0, 1: 0},
#                 "NBPAYMENTSDATA_METHOD_EXC": {0: 0, 1: 0},
#                 "NBANNUITYDATA_SYSTEMATICWITHDRAWAL_VALUE_False": {0: 1, 1: 1},
#                 "NBANNUITYDATA_SYSTEMATICWITHDRAWAL_VALUE_True": {0: 0, 1: 0},
#                 "NBREPLACEMENTINFODATA_EXCHANGETYPE_VALUE_External": {0: 0, 1: 0},
#                 "NBREPLACEMENTINFODATA_EXCHANGETYPE_VALUE_Internal": {0: 0, 1: 0},
#                 "NBANNUITYDATA_ACCOUNTTYPE_VALUE_Checking": {0: 0, 1: 0},
#                 "NBANNUITYDATA_ACCOUNTTYPE_VALUE_Savings": {0: 0, 1: 0},
#                 "NBANNUITYDATA_CHECKFORMS_VALUE_False": {0: 0, 1: 0},
#                 "NBANNUITYDATA_CHECKFORMS_VALUE_True": {0: 1, 1: 1},
#                 "NBANNUITYDATA_ROLLOVER_VALUE_False": {0: 1, 1: 1},
#                 "NBANNUITYDATA_ROLLOVER_VALUE_True": {0: 0, 1: 0},
#                 "NBANNUITYDATA_QUALIFIED_VALUE_False": {0: 0, 1: 1},
#                 "NBANNUITYDATA_QUALIFIED_VALUE_True": {0: 1, 1: 0},
#                 "NBAGENTDATA_ISPRIMARY_VALUE_False": {0: 0, 1: 0},
#                 "NBAGENTDATA_ISPRIMARY_VALUE_True": {0: 1, 1: 1},
#                 "NBREPLACEMENTINFODATA_ACCTYPE_VALUE_Annuity ": {0: 0, 1: 0},
#                 "NBREPLACEMENTINFODATA_ACCTYPE_VALUE_Life Insurance": {0: 0, 1: 0},
#                 "NBREPLACEMENTINFODATA_ACCTYPE_VALUE_Other": {0: 0, 1: 0},
#                 "NBREPLACEMENTINFODATA_PREVPLANTYPE_VALUE_401(k)": {0: 0, 1: 0},
#                 "NBREPLACEMENTINFODATA_PREVPLANTYPE_VALUE_403(b)": {0: 0, 1: 0},
#                 "NBREPLACEMENTINFODATA_PREVPLANTYPE_VALUE_457 Deferred Comp.": {
#                     0: 0,
#                     1: 0,
#                 },
#                 "NBREPLACEMENTINFODATA_PREVPLANTYPE_VALUE_Beneficiary IRA": {
#                     0: 0,
#                     1: 0,
#                 },
#                 "NBREPLACEMENTINFODATA_PREVPLANTYPE_VALUE_Corporate Pension": {
#                     0: 0,
#                     1: 0,
#                 },
#                 "NBREPLACEMENTINFODATA_PREVPLANTYPE_VALUE_Defined Benefit": {
#                     0: 0,
#                     1: 0,
#                 },
#                 "NBREPLACEMENTINFODATA_PREVPLANTYPE_VALUE_Group TSA": {0: 0, 1: 0},
#                 "NBREPLACEMENTINFODATA_PREVPLANTYPE_VALUE_IRA-Rollover": {0: 0, 1: 0},
#                 "NBREPLACEMENTINFODATA_PREVPLANTYPE_VALUE_IRA-Simple": {0: 0, 1: 0},
#                 "NBREPLACEMENTINFODATA_PREVPLANTYPE_VALUE_Inheritance IRA (added by ERA)": {
#                     0: 0,
#                     1: 0,
#                 },
#                 "NBREPLACEMENTINFODATA_PREVPLANTYPE_VALUE_Inherited Non-Qualified": {
#                     0: 0,
#                     1: 0,
#                 },
#                 "NBREPLACEMENTINFODATA_PREVPLANTYPE_VALUE_Non-Qualified": {0: 0, 1: 0},
#                 "NBREPLACEMENTINFODATA_PREVPLANTYPE_VALUE_Not specified": {0: 0, 1: 0},
#                 "NBREPLACEMENTINFODATA_PREVPLANTYPE_VALUE_Pension Trust": {0: 0, 1: 0},
#                 "NBREPLACEMENTINFODATA_PREVPLANTYPE_VALUE_Roth IRA": {0: 0, 1: 0},
#                 "NBREPLACEMENTINFODATA_PREVPLANTYPE_VALUE_SEP IRA": {0: 0, 1: 0},
#                 "NBREPLACEMENTINFODATA_PREVPLANTYPE_VALUE_Traditional IRA": {
#                     0: 0,
#                     1: 0,
#                 },
#                 "NBREPLACEMENTINFODATA_PREVPLANTYPE_VALUE_Trust": {0: 0, 1: 0},
#                 "NBREPLACEMENTINFODATA_PREVPLANTYPE_VALUE_IRA-Spousal": {0: 0, 1: 0},
#                 "NBREPLACEMENTINFODATA_PREVPLANTYPE_VALUE_401(a)": {0: 0, 1: 0},
#                 "NBREPLACEMENTINFODATA_PREVPLANTYPE_VALUE_401(a) - Sched A": {
#                     0: 0,
#                     1: 0,
#                 },
#                 "NBREPLACEMENTINFODATA_PREVPLANTYPE_VALUE_Roth 401(k)": {0: 0, 1: 0},
#                 "NBREPLACEMENTINFODATA_PREVPLANTYPE_VALUE_Converted Roth IRA": {
#                     0: 0,
#                     1: 0,
#                 },
#                 "NBREPLACEMENTINFODATA_PREVPLANTYPE_VALUE_Life Insurance": {0: 0, 1: 0},
#                 "NBREPLACEMENTINFODATA_PREVPLANTYPE_VALUE_Roth 403(b)": {0: 0, 1: 0},
#                 "NBREPLACEMENTINFODATA_PREVPLANTYPE_VALUE_Custodial IRA": {0: 0, 1: 0},
#                 "NBREPLACEMENTINFODATA_PREVPLANTYPE_VALUE_VUL NQ": {0: 0, 1: 0},
#                 "NBANNUITYDATA_EXPECTEDSURRPENALTY_VALUE_No": {0: 1, 1: 1},
#                 "NBANNUITYDATA_EXPECTEDSURRPENALTY_VALUE_Yes": {0: 0, 1: 0},
#                 "CASEHDRDATA_ISGROUP_VALUE_False": {0: 0, 1: 0},
#                 "CASEHDRDATA_ISGROUP_VALUE_True": {0: 1, 1: 1},
#                 "CASEHDRDATA_STATUS_VALUE_Completed": {0: 0, 1: 0},
#                 "CASEHDRDATA_STATUS_VALUE_Declined": {0: 0, 1: 0},
#                 "CASEHDRDATA_STATUS_VALUE_Issued": {0: 1, 1: 0},
#                 "CASEHDRDATA_STATUS_VALUE_New": {0: 0, 1: 1},
#                 "CASEHDRDATA_STATUS_VALUE_Pending Re-Issue": {0: 0, 1: 0},
#                 "CASEHDRDATA_STATUS_VALUE_Pending": {0: 0, 1: 0},
#                 "CASEHDRDATA_UNDERWRITINGSTATUS_VALUE_Completed": {0: 0, 1: 0},
#                 "CASEHDRDATA_UNDERWRITINGSTATUS_VALUE_Last Requirement": {0: 1, 1: 0},
#                 "CASEHDRDATA_UNDERWRITINGSTATUS_VALUE_Requirement Review": {0: 0, 1: 1},
#                 "CASEHDRDATA_UNDERWRITINGSTATUS_VALUE_Waiting/Pending": {0: 0, 1: 0},
#                 "FINANCIALGOALSDATA_EXPECTEDSURRPENALTY_CODE_Blank": {0: 0, 1: 0},
#                 "FINANCIALGOALSDATA_EXPECTEDSURRPENALTY_CODE_No": {0: 1, 1: 1},
#                 "FINANCIALGOALSDATA_EXPECTEDSURRPENALTY_CODE_Yes": {0: 0, 1: 0},
#                 "FINANCIALGOALSDATA_REVERSEMORTGAGE_CODE_Blank": {0: 0, 1: 0},
#                 "FINANCIALGOALSDATA_REVERSEMORTGAGE_CODE_No": {0: 1, 1: 1},
#                 "FINANCIALGOALSDATA_REVERSEMORTGAGE_CODE_Yes": {0: 0, 1: 0},
#                 "CASEHDRDATA_ESIGNATUREIND_VALUE_False": {0: 0, 1: 0},
#                 "CASEHDRDATA_ESIGNATUREIND_VALUE_True": {0: 1, 1: 1},
#                 "FINANCIALGOALSDATA_ADEQUATEASSETS_VALUE_False": {0: 0, 1: 0},
#                 "FINANCIALGOALSDATA_ADEQUATEASSETS_VALUE_True": {0: 1, 1: 1},
#                 "APPCOVERAGESDATA_PRODMODELID_VALUE_RSL - Deferral": {0: 0, 1: 0},
#                 "class": {0: 1, 1: 0},
#             }
#         },
#     )
# )


from ryu.base import app_manager
from ryu.controller import ofp_event
from ryu.controller.handler import CONFIG_DISPATCHER, MAIN_DISPATCHER
from ryu.controller.handler import set_ev_cls
from ryu.ofproto import ofproto_v1_3
from ryu.lib.packet import packet
from ryu.lib.packet import ethernet
from ryu.lib.packet import ether_types
from ryu import cfg
CONF = cfg.CONF
from libs.ofProtoHelper.ofprotoHelper import ofProtoHelper

class L2Handler(app_manager.RyuApp):
    OFP_VERSIONS = [ofproto_v1_3.OFP_VERSION]
    opts = [
        cfg.IntOpt('priority',
                default=1,
                help='Goto priority in table 0'),
        cfg.IntOpt('table',
               default=2,
               help='Table to use for L2 switching'),
        cfg.IntOpt('cookie_arp',
                default=101,
                help='FLow mod cookie to use for Controller event on arp'),
        cfg.IntOpt('cookie_low',
                default=101,
                help='FLow mod cookie to use for Controller event low value'),
        cfg.IntOpt('cookie_high',
                default=101,
                help='FLow mod cookie to use for Controller event high value')
    ]

    def __init__(self, *args, **kwargs):
        super(L2Handler, self).__init__(*args, **kwargs)

        self.mac_to_port = {}

        CONF.register_opts(self.opts, group='l2')
        self.ofHelper = ofProtoHelper()

    @set_ev_cls(ofp_event.EventOFPSwitchFeatures, CONFIG_DISPATCHER)
    def switch_features_handler(self, ev):
        datapath = ev.msg.datapath
        ofproto = datapath.ofproto
        parser = datapath.ofproto_parser

        match = parser.OFPMatch()
        actions = [parser.OFPActionOutput(ofproto.OFPP_CONTROLLER,
                                          ofproto.OFPCML_NO_BUFFER)]

        self.ofHelper.add_goto(datapath, CONF.l2.priority, match, 0, CONF.l2.table)
        self.ofHelper.add_flow(datapath, 0, match, actions, CONF.l2.table, cookie=CONF.l2.cookie_arp)

    @set_ev_cls(ofp_event.EventOFPPacketIn, MAIN_DISPATCHER)
    def _packet_in_handler(self, ev):
        if ev.msg.cookie in range(CONF.l2.cookie_low, CONF.l2.cookie_high):
            self.logger.debug('packet being handled in L2 module')
        else:
            return
        # If you hit this you might want to increase
        # the "miss_send_length" of your switch
        if ev.msg.msg_len < ev.msg.total_len:
            self.logger.debug("packet truncated: only %s of %s bytes",
                              ev.msg.msg_len, ev.msg.total_len)
        msg = ev.msg
        datapath = msg.datapath
        ofproto = datapath.ofproto
        parser = datapath.ofproto_parser
        in_port = msg.match['in_port']

        pkt = packet.Packet(msg.data)
        eth = pkt.get_protocols(ethernet.ethernet)[0]

        if eth.ethertype == ether_types.ETH_TYPE_LLDP:
            # ignore lldp packet
            return
        dst = eth.dst
        src = eth.src

        dpid = datapath.id
        self.mac_to_port.setdefault(dpid, {})

        self.logger.info("packet in %s %s %s %s", dpid, src, dst, in_port)

        # learn a mac address to avoid FLOOD next time.
        self.mac_to_port[dpid][src] = in_port

        if dst in self.mac_to_port[dpid]:
            out_port = self.mac_to_port[dpid][dst]
        else:
            out_port = ofproto.OFPP_FLOOD

        actions = [parser.OFPActionOutput(out_port)]

        # install a flow to avoid packet_in next time
        if out_port != ofproto.OFPP_FLOOD:
            match = parser.OFPMatch(in_port=in_port, eth_dst=dst, eth_src=src)
            # verify if we have a valid buffer_id, if yes avoid to send both
            # flow_mod & packet_out
            if msg.buffer_id != ofproto.OFP_NO_BUFFER:
                self.ofHelper.add_flow(datapath, 1, match, actions, CONF.l2.table, buffer_id=msg.buffer_id)
                return
            else:
                self.ofHelper.add_flow(datapath, 1, match, actions, CONF.l2.table)
        data = None
        if msg.buffer_id == ofproto.OFP_NO_BUFFER:
            data = msg.data

        out = parser.OFPPacketOut(datapath=datapath, buffer_id=msg.buffer_id,
                                  in_port=in_port, actions=actions, data=data)
        datapath.send_msg(out)


#!/usr/bin/python
import sys
from datetime import datetime

for line in sys.stdin:
    line = line.strip().split(',')
    if line[0] == 'VendorID' or line[0] == 'WBAN' or line[0]=='"Date/Time"' or line[0]=="Dispatching_base_num" or line[0] == 'vendor_id':
        continue
    if len(line) == 18 or len(line) == 19:
        tpep_pickup_datetime = line[1]
        year_month_day = datetime.strptime(tpep_pickup_datetime, '%Y-%m-%d %H:%M:%S').strftime('%Y-%m-%d')
        #total_amount = line[18]
        print(("%s\t%d,y") % (year_month_day, 1))
    elif len(line)== 4:
        if line[0].isalnum():
            pickup_datetime = line[1]
            pickup_date = datetime.strptime(pickup_datetime, '%Y-%m-%d %H:%M:%S').strftime('%Y-%m-%d')
            print(("%s\t%d,u") % (pickup_date, 1))
        else:
            pickup_datetime = line[0]
            pickup_datetime = pickup_datetime[1:-1]
            pickup_date = datetime.strptime(pickup_datetime, '%m/%d/%Y %H:%M:%S').strftime('%Y-%m-%d')
            print(("%s\t%d,u") % (pickup_date, 1))

    elif len(line) == 21 or len(line) == 22 or len(line) == 23:
        lpep_pickup_datetime = line[1]
        year_month_day = datetime.strptime(lpep_pickup_datetime, '%Y-%m-%d %H:%M:%S').strftime('%Y-%m-%d')
        #total_amount = line[18]
        print(("%s\t%d,g") % (year_month_day, 1))
    elif len(line) == 50:
        year_month_day = datetime.strptime(line[1], '%Y%m%d').strftime('%Y-%m-%d')
        t_max = line[2]
        t_min = line[4]
        t_avg = line[6]
        weather_type = line[22]
        depth = line[24]
        water1 = line[26]
        snowfall = line[28]
        precip_total = line[30]
        avg_speed = line[40]
        print(("%s\t%s,%s,%s,%s,%s,%s,%s,%s,%s") % (year_month_day, t_max, t_min, t_avg, weather_type, depth, water1, snowfall, precip_total, avg_speed))


from datetime import date, timedelta

import pytest
from example import print_owing

from invoice import Invoice
from order import Order

@pytest.fixture
def invoice():
    _invoice = Invoice("Minwoo Jeong")
    for amount in range(10, 60, 10):
        _invoice.add_order(Order(amount))
    due_date = date.today() + timedelta(days=30)
    _invoice.due_date = due_date

    return _invoice

def test_print_owing(capsys, invoice):
    print_owing(invoice)
    expected = (
        "**** Customer Outstanding Debt ****\n"
        f"Customer: {invoice.customer}\n"
        "Outstanding Debt: 150\n"
        f"Due Date: {invoice.due_date.strftime('%d/%m/%y')}\n"
    )
    captured = capsys.readouterr()
    assert captured.out == expected


"""
Resource normalizer
"""
from collections import namedtuple

from prjname.common import exceptions
import collections

RULE_NT = namedtuple('Rule', ['execute', 'args', 'default'])

def Rule(args, execute=None, default=None):                             # pylint: disable=C0103
    """
    Returns a named tuple that represent a transformation rule
    @param args set of arguments to be used by the rule
    @param execute transformation function who receive the args
    @param default value used when the attribute defined by path is not in the resource
    """

    return RULE_NT(execute, args, default)

def normalize_resources(resources, normalization_rules, is_ordered=False):
    """
    Normalize a list of resources based on a set of rules
    @param resources list of resources to be normalized
    @param normalization_rules set of rules to normalize the resources
    @param is_ordered mantain the order specified in the schema
    """

    normalized_resources = []
    for resource in resources:
        normalized_resource = normalize_resource(resource,
                                                 normalization_rules,
                                                 is_ordered)
        normalized_resources.append(normalized_resource)

    return normalized_resources

def normalize_resource(resource, normalization_rules, is_ordered=False):
    """
    Normalize a resource based on a set of rules
    @param resources list of resources to be normalized
    @param normalization_rules set of rules to normalize the resources
    @param is_ordered mantain the order specified in the schema
    """

    normalized_resource = {}

    for resource_path, rule in normalization_rules.iteritems():
        if isinstance(rule, dict):
            normalized_value = normalize_resource(resource,
                                                  rule,
                                                  is_ordered)
        else:
            normalized_value = _normalize(rule, resource)

        _set_resource_value_based_on_dotted_path(resource_path,
                                                 normalized_value,
                                                 normalized_resource)

    return normalized_resource

def normalize_response(response, normalization_rules=None):
    """
    Normalize the external service response using the normalization rules
    """
    # pylint: disable=R0201
    normalizer_mapping = {
        dict: normalize_resource,
        list: normalize_resources,
        collections.OrderedDict: normalize_resource
    }

    if normalization_rules is None:
        return response

    try:
        normalized_response = normalizer_mapping[type(response)](
            response, normalization_rules)
    except KeyError as ex:
        raise ValueError('Normalization error: %s' % ex)

    return normalized_response

# Helper method

def _normalize(rule, resource):
    """
    Apply a transformation rule to the resource
    @param rule transformation rule to be applied to the resource
    @param resource resource to transform
    """

    execute = rule.execute
    args = rule.args
    default = rule.default

    kwargs = {}
    for arg in args:
        kwargs[arg.split('.')[-1]] = _get_resource_value_based_on_dotted_path(
            arg, resource, default if not execute else None)

    if execute:
        try:
            normalized_value = execute(**kwargs)
        except Exception:  # pylint: disable=W0703
            normalized_value = default
    elif len(kwargs) > 1:
        raise exceptions.InvalidArgument(
            'too many args for a rule with no "execute"')
    else:
        normalized_key, normalized_value = kwargs.popitem()  # pylint: disable=W0612

    return normalized_value

def _set_resource_value_based_on_dotted_path(path, value, resource):  # pylint: disable=C0103
    """
    Set a resource value based on a dotted path
    @param path string containing a dotted path to an attribute
    @param value value to be set to the attribute specified by path
    @param resource resource who contains the attribute to be changed
    """

    splitted_path = path.split('.')

    for index, path_key in enumerate(splitted_path):
        if index == len(splitted_path) - 1:
            resource[path_key] = value
        else:
            resource = resource[path_key]

def _get_resource_value_based_on_dotted_path(path, resource, default):  # pylint: disable=C0103
    """
    Get a resource value based on a dotted path
    @param path string containing a dotted path to an attribute
    @param resource resource who contains the attribute to be returned
    @param default value used when the attribute defined by path is not in the resource
    """

    splitted_path = path.split('.')

    try:
        for path_key in splitted_path:
            resource = resource.get(path_key, default)
    except AttributeError:
        pass

    return resource


import vitalfile
import os
import shutil

idir = r'\\Vitalnew\snuhor\ex8_note'
for path, dir, files in os.walk(idir):
    for filename in files:
        filepath = "{}/{}".format(path, filename)
        print(filename, end='\t')
        vit = vitalfile.VitalFile(filepath)
        print(int(vit.dtstart))


from typing import Mapping

import pandas as pd

from cin_validator.rule_engine import CINTable, RuleContext, rule_definition
from cin_validator.test_engine import run_rule

CINdetails = CINTable.CINdetails
LAchildID = CINdetails.LAchildID
ReferralNFA = CINdetails.ReferralNFA
PrimaryNeedCode = CINdetails.PrimaryNeedCode
CINdetailsID = CINdetails.CINdetailsID

@rule_definition(
    code="8610",
    module=CINTable.CINdetails,
    message="Primary Need code is missing for a referral which led to further action.",
    affected_fields=[ReferralNFA, PrimaryNeedCode],
)
def validate(
    data_container: Mapping[CINTable, pd.DataFrame], rule_context: RuleContext
):
    df = data_container[CINdetails]
    df.index.name = "ROW_ID"
    df.reset_index(inplace=True)

    # If <ReferralNFA> (N00112) = false or 0
    # then
    # <PrimaryNeedCode> (N00101) must be present

    condition = (df[ReferralNFA].isin(["false", "0"])) & (df[PrimaryNeedCode].isna())
    df_issues = df[condition].reset_index()

    link_id = tuple(zip(df_issues[LAchildID], df_issues[CINdetailsID]))
    df_issues["ERROR_ID"] = link_id
    df_issues = (
        df_issues.groupby("ERROR_ID", group_keys=False)["ROW_ID"]
        .apply(list)
        .reset_index()
    )
    rule_context.push_type_1(
        table=CINdetails, columns=[ReferralNFA, PrimaryNeedCode], row_df=df_issues
    )

def test_validate():
    #  Fails rows 0, 1, and 3
    sample_cin = pd.DataFrame(
        {
            "LAchildID": ["child1", "child2", "child3", "child4", "child5"],
            "ReferralNFA": [
                "false",
                "0",
                "1",
                pd.NA,
                "true",
            ],
            "PrimaryNeedCode": [
                pd.NA,  # fail
                pd.NA,  # fail
                "12/09/2022",
                "05/12/1997",
                pd.NA,  # ignore: ReferralNFA is true
            ],
            "CINdetailsID": [
                "ID1",
                "ID2",
                "ID3",
                "ID4",
                "ID5",
            ],
        }
    )

    result = run_rule(validate, {CINdetails: sample_cin})

    issues = result.type1_issues

    issue_table = issues.table
    assert issue_table == CINdetails
    issue_columns = issues.columns
    assert issue_columns == [ReferralNFA, PrimaryNeedCode]

    issue_rows = issues.row_df

    assert len(issue_rows) == 2
    assert isinstance(issue_rows, pd.DataFrame)
    assert issue_rows.columns.to_list() == ["ERROR_ID", "ROW_ID"]

    expected_df = pd.DataFrame(
        [
            {
                "ERROR_ID": (
                    "child1",
                    "ID1",
                ),
                "ROW_ID": [0],
            },
            {
                "ERROR_ID": ("child2", "ID2"),
                "ROW_ID": [1],
            },
        ]
    )
    assert issue_rows.equals(expected_df)

    assert result.definition.code == "8610"
    assert (
        result.definition.message
        == "Primary Need code is missing for a referral which led to further action."
    )


import pygame
pygame.display.set_mode()
sprite_sheet = pygame.image.load('./textures/mario/mario_sprites.png').convert_alpha()
sprite_width = sprite_sheet.get_width()//14
sprite_height = sprite_sheet.get_height()//12
sprites_per_row = sprite_sheet.get_width() // sprite_width
sprites_per_col = sprite_sheet.get_height() // sprite_height
frames = []
count = 0
for row in range(sprites_per_col):
    for col in range(sprites_per_row):
        x = col * sprite_width
        y = row * sprite_height
        sprite = sprite_sheet.subsurface((x, y, sprite_width, sprite_height))
        pygame.image.save(sprite,f"./textures/mario/{count}.png")
        count += 1


import random
from rock_paper_scissors import ROCK, PAPER, SCISSORS, BasePlayer

class Player(BasePlayer):

    player_name = 'Monty2'
    author = 'Leonard B'
    x_beats_y = {ROCK: SCISSORS, PAPER: ROCK, SCISSORS: PAPER}

    def __init__(self):
        self.history = []
        self.current_choice = None
        self.counts = {ROCK: 0, PAPER: 0, SCISSORS: 0}
        self.round = 1

    def is_win(self, my_play, other_play):
        for key, value in self.x_beats_y.items():
            if key == my_play and value == other_play:
                return True
        return False

    def play(self):
        """
        Must return a single var for the decided play:
        either 'ROCK', 'PAPER', 'SCISSORS'
        """
        if self.round == 1:
            self.current_choice = random.choice([ROCK, PAPER, SCISSORS])
        elif not self.is_win(self.current_choice, self.history[-1]):
            self.current_choice = random.choice([ROCK, PAPER, SCISSORS])
        else:
            random_number = random.randint(1, self.round)
            if random_number <= self.counts[ROCK]:
                self.current_choice = PAPER
            elif random_number <= self.counts[ROCK] + self.counts[PAPER]:
                self.current_choice = SCISSORS
            else:
                self.current_choice = ROCK
        return self.current_choice

    def result(self, their_play):
        self.history.append((self.current_choice, their_play))
        self.counts[their_play] += 1
        self.round += 1


from vpython import *
import numpy as np

arrowL = 2
arrowT = 0.02

xArrow = arrow(axis=vector(1,0,0), color=color.red, length=arrowL, shaftwidth=arrowT)
yArrow = arrow(axis=vector(0,1,0), color=color.green, length=arrowL, shaftwidth=arrowT)
zArrow = arrow(axis=vector(0,0,1), color=color.blue, length=arrowL, shaftwidth=arrowT)

theta = 0

pntArrow = arrow(axis=vector(arrowL*np.cos(theta), arrowL*np.sin(theta), 0), color=color.orange, length=arrowL, shaftwidth=arrowT)

flag = 0

# while True:
#     for theta in np.linspace(0, 2*np.pi, 1000):
#         rate(50)
#         pntArrow.axis = vector(arrowL*np.cos(theta), arrowL*np.sin(theta), 0)
#         pntArrow.length = arrowL

while True:
    if flag%3 == 0:
        for theta in np.linspace(0, 2*np.pi, 1000):
            rate(150)
            pntArrow.axis = vector(arrowL*np.cos(theta), arrowL*np.sin(theta), 0)
            pntArrow.length = arrowL

    if flag%3 == 1:
        for theta in np.linspace(0, 2*np.pi, 1000):
            rate(150)
            pntArrow.axis = vector(arrowL*np.sin(theta), 0,arrowL*np.cos(theta))
            pntArrow.length = arrowL

    if flag%3 == 2:
        for theta in np.linspace(0, 2*np.pi, 1000):
            rate(150)
            pntArrow.axis = vector(0, arrowL*np.cos(theta), arrowL*np.sin(theta))
            pntArrow.length = arrowL

    flag =flag + 1


# -*- coding: utf-8 -*-

# Name of the environment variable used to store the 'Consumer Key'
KAYAK_KEY = 'KAYAK_CONSUMER_KEY'

# environment variable to store the 'Consumer Secret'
KAYAK_SECRET = 'KAYAK_CONSUMER_SECRET'

# header prefix used in different types of authentications
AUTH_HEADER_PREFIX = 'Basic'
BEARER_AUTH_HEADER_PREFIX = 'Bearer'

# request attributes
REQ_USER_AGENT = 'Kayak@hackpravj-v0.0.9'
REQ_ENCODING = 'gzip'
REQ_CONTENT_TYPE = 'application/x-www-form-urlencoded;charset=UTF-8'
REQ_DATA = {'grant_type': 'client_credentials'}

# URLs to 'request' a new token or 'invalidate' an existing token
REQUEST_TOKEN_URL = 'https://api.twitter.com/oauth2/token'
INVALIDATE_TOKEN_URL = 'https://api.twitter.com/oauth2/invalidate_token'

# key-value pair to match in order to verify correctness of response
VERIFICATION_KEY = 'token_type'
VERIFICATION_VALUE = 'bearer'

# key name associated with the token
TOKEN_KEY = 'access_token'

# URL for Twitter search API
SEARCH_API_URL = 'https://api.twitter.com/1.1/search/tweets.json'

# Keys to look for in 'status' response
# in case of breaking changes in Twitter API results
TWEET_TEXT_KEY = 'text'
TWEET_ID_KEY = 'id'
TWEET_RETWEET_KEY = 'retweet_count'

# special parameters to work with Twitter timelines
# https://dev.twitter.com/rest/public/timelines
TWEET_SINCE_ID = 'since_id'
TWEET_MAX_ID = 'max_id'

__all__ = [
    KAYAK_KEY,
    KAYAK_SECRET,
    AUTH_HEADER_PREFIX,
    BEARER_AUTH_HEADER_PREFIX,
    REQ_USER_AGENT,
    REQ_ENCODING,
    REQ_CONTENT_TYPE,
    REQ_DATA,
    REQUEST_TOKEN_URL,
    INVALIDATE_TOKEN_URL,
    VERIFICATION_KEY,
    VERIFICATION_VALUE,
    TOKEN_KEY,
    SEARCH_API_URL,
    TWEET_TEXT_KEY,
    TWEET_ID_KEY,
    TWEET_RETWEET_KEY,
    TWEET_SINCE_ID]


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Tue Jun 12 17:09:30 2018

@author: abauville
"""
#import sys
#sys.path.insert(0, '../../src/UserInput')
import matplotlib.pyplot as plt
from matplotlib.colors import LinearSegmentedColormap
import numpy as np
import OutputDef as Output

def get_XYandPattern(dataFolder,lc=2.0e3,sampleRate=1, nLayersX = 0, nLayersY=7,minStrain=0.1,maxStrain=5.0,xmin = 'auto',xmax = 'auto',ymin='auto',ymax='auto', mainDir='x'):
    ## Load and subsample
    # ================================
    PartX  = Output.getParticleData(dataFolder + 'particles_x.bin',True).data[0::sampleRate]/lc
    PartY  = Output.getParticleData(dataFolder + 'particles_y.bin',True).data[0::sampleRate]/lc
    PartXIni  = Output.getParticleData(dataFolder + 'particles_xIni.bin',True).data[0::sampleRate]/lc
    PartYIni  = Output.getParticleData(dataFolder + 'particles_yIni.bin',True).data[0::sampleRate]/lc

    PartStrain  = Output.getParticleData(dataFolder + 'particles_strain.bin',True).data[0::sampleRate]

    if xmin=='auto':
        xmin = np.min(PartX)
    if xmax=='auto':
        xmax = np.max(PartX)
    if ymin=='auto':
        ymin = np.min(PartY)
    if ymax=='auto':
        ymax = np.max(PartY)

    ## Geometrical pattern
    # ================================
#    nLayersY = 7
#    nLayersX = 16
#    xmax = 0.0
#    xmin = -32.0#np.min(PartXIni)

    if mainDir == 'x':
        if nLayersX > 0:
            YPattern = PartYIni
            YPattern -= ymin
            YPattern /= ymax-ymin
    #        YPattern = np.cos(PartYIni*1.0*np.pi*nLayersY+np.pi/2.0)
            YPattern = np.cos(YPattern*1.0*np.pi*nLayersY+np.pi/2.0)
            maxVal= np.max(YPattern)
            YPattern = np.round((YPattern+maxVal)/(2.0*maxVal))

            nColors = nLayersX

            XPattern = PartXIni
#            XPattern -= xmin
            XPattern /= xmax-xmin

            PartPattern= 4.0*np.floor(-XPattern * (nLayersX) )
    #

#
            PartPattern+= 2.0*YPattern
            PartPattern = PartPattern%(4*nLayersX)

        else:
            PartPattern = np.zeros(PartStrain.shape)
            nColors = 1
    elif mainDir=='y':
        if nLayersY>0:
            PartPattern = np.ones(PartStrain.shape)
            nColors = nLayersY

            if nLayersX<=1:
                nLayersX=0

            XPattern = PartXIni
            XPattern -= xmin
            XPattern /= xmax-xmin
            XPattern = np.cos(PartXIni*1.0*np.pi*nLayersX+np.pi/2.0)
            maxVal= np.max(XPattern)

            if nLayersX>1:
                XPattern = np.round((XPattern+maxVal)/(2.0*maxVal))
            else:
                XPattern = np.round((XPattern)/(2.0*maxVal))
#            XPattern = np.round((XPattern)/(2.0*maxVal))
#

            YPattern = PartYIni
            YPattern /= ymax-ymin
#            YPattern = -YPattern

#            YPattern[YPattern>0.95] = 6.0
#            nColors = 6

            PartPattern= 4.0*np.floor(YPattern * (nLayersY) )
            PartPattern+= 2.0*XPattern
            PartPattern = PartPattern%(4*nLayersY)

        else:
            PartPattern = np.zeros(PartStrain.shape)
            nColors = 1
    else:
        raise ValueError("mainDir can take the values 'x' or 'y' only.")

    ## Strain pattern
    # ================================
    PartStrain -= minStrain
    PartStrain[PartStrain<0.0] = 0.0
    PartStrain /= maxStrain-minStrain

    PartStrainLogical = PartStrain>1.0
    PartStrain[PartStrainLogical] = 1.0

    PartPattern += PartStrain

    return (PartX, PartY, PartPattern, nColors)

def getColormap(nColors,name='KellyMod_Layers_Strain',renderer=0,CMAP=np.array([]),
                darknessFactor=[1.0,0.0,0.5,0.0],
                RGBShift = [[0.0,0.0,0.0], # Main color
                            [0.0,0.0,0.0], # Main Color strain
                            [0.0,0.0,0.0], # Minor Color
                            [0.0,0.0,0.0]], # Minor Color strain
                ):
    # Darkness Factor is applied before RGB shift

    ## Kenneth Kelly's 22 colour of maximum contrast
#    nColors  = 22

    if type(CMAP) is list:
        CMAP=np.array(CMAP)

    if (CMAP.size==0):
        if nColors>22:
            raise ValueError("error: nColors should not exceed 22")
        condensedCMAPtemp = np.zeros((nColors,4))
        for i in range (nColors):
            if i ==  0: condensedCMAPtemp[ i,:] = [242, 243, 244, 255]
            if i ==  1: condensedCMAPtemp[ i,:] = [ 34,  34,  34, 255]
            if i ==  2: condensedCMAPtemp[ i,:] = [243, 195,   0, 255]
            if i ==  3: condensedCMAPtemp[ i,:] = [135,  86, 146,255]
            if i ==  4: condensedCMAPtemp[ i,:] = [243, 132,   0,255]
            if i ==  5: condensedCMAPtemp[ i,:] = [161, 202, 241,255]
            if i ==  6: condensedCMAPtemp[ i,:] = [190,   0,  50,255]
    #        if i ==  7: condensedCMAPtemp[ i,:] = [194, 178, 128,255]
            if i ==  7: condensedCMAPtemp[ i,:] = [ 50, 178, 128,255]
    #        if i ==  8: condensedCMAPtemp[ i,:] = [132, 132, 130,255]
            if i ==  8: condensedCMAPtemp[ i,:] = [162,  80, 200,255]
            if i ==  9: condensedCMAPtemp[ i,:] = [  0, 136,  86,255]
            if i == 10: condensedCMAPtemp[ i,:] = [230, 143, 172,255]
            if i == 11: condensedCMAPtemp[ i,:] = [  0, 103, 165,255]
            if i == 12: condensedCMAPtemp[ i,:] = [249, 147, 121,255]
            if i == 13: condensedCMAPtemp[ i,:] = [ 96,  78, 151,255]
            if i == 14: condensedCMAPtemp[ i,:] = [246, 166,   0,255]
            if i == 15: condensedCMAPtemp[ i,:] = [179,  68, 108,255]
            if i == 16: condensedCMAPtemp[ i,:] = [220, 211,   0,255]
            if i == 17: condensedCMAPtemp[ i,:] = [136,  45,  23,255]
            if i == 18: condensedCMAPtemp[ i,:] = [141, 182,   0,255]
            if i == 19: condensedCMAPtemp[ i,:] = [101,  69,  34,255]
            if i == 20: condensedCMAPtemp[ i,:] = [226,  88,  34,255]
            if i == 21: condensedCMAPtemp[ i,:] = [ 43,  61,  38,255]

        condensedCMAPtemp /= 255.0
        #CMAP = LinearSegmentedColormap.from_list('Kelly',condensedCMAPtemp)
        #plt.register_cmap(cmap=CMAP)

        condensedCMAPtemp[:,0:2] *= .5
        condensedCMAPtemp[:,0:2] += .1
#        condensedCMAPtemp[:,0  ] += .4
#        condensedCMAPtemp[:,1  ] += .35

#        condensedCMAPtemp[:,0:3] *= 0
#        condensedCMAPtemp[:,0:3] += 1.0
#        condensedCMAPtemp = np.ones((nColors,4))
        condensedCMAPtemp[:,0  ] += .4
        condensedCMAPtemp[:,1  ] += .35

#

    else:
        try:
            if (CMAP.shape[1]!=4):
                raise ValueError("error in getColormap: CMAP must have dimensions nColors*4");
        except IndexError: # Occurs if CMAP contains only one color CMAP.shape = (4,)
            if (CMAP.shape[0]!=4):
                raise ValueError("error in getColormap: CMAP must have dimensions nColors*4");
        nColors = CMAP.shape[0]
        condensedCMAPtemp = CMAP

    condensedCMAPtemp[condensedCMAPtemp>1.0] = 1.0
    condensedCMAPtemp[condensedCMAPtemp<0.0] = 0.0

    condensedCMAP = np.zeros((4*nColors,4))
    for i in range(4):
        if i%2==0:
            condensedCMAP[i::4,:] = condensedCMAPtemp*darknessFactor[i] #* 0.4 + .3
        else:
            condensedCMAP[i::4,:] = condensedCMAP[i-1::4,:]*darknessFactor[i] #* 0.4 + .3

        condensedCMAP[i::4,0] += RGBShift[i][0]  # Shift Red
        condensedCMAP[i::4,1] += RGBShift[i][1]  # Shift Green
        condensedCMAP[i::4,2] += RGBShift[i][2]  # Shift Blue

    condensedCMAP[condensedCMAP>1.0] = 1.0
    condensedCMAP[condensedCMAP<0.0] = 0.0
    condensedCMAP[:,-1] = 1.0
#    nColors = condensedCMAP.shape[0]

    if renderer == 0: # renderer is Matplotlib
        res = 12
        CMAP = LinearSegmentedColormap.from_list(name,condensedCMAP,N=res*4*nColors-(res-1))
        return CMAP
        #plt.register_cmap(cmap=CMAP)

    if renderer == 1: # renderer is Mayavi
        CMAP = np.zeros((255,4))

        n = condensedCMAP.shape[0]
        for i in range(n-1):
            iS = int(round(255 *  i/(n-1) ))
            iE = int(round(255 * (i+1)/(n-1)))
            for j in range(4):
                CMAP[iS:iE,j] = np.linspace(condensedCMAP[i,j],condensedCMAP[i+1,j],int(iE-iS))

        CMAP = (CMAP*255.0).astype("uint8")
        return CMAP


import pickle
import random
from Prediction.RobModel import RobModel
import numpy as np
import db_config
from matplotlib import pyplot
from keras.models import Sequential
from keras.layers import Dense
from keras.layers import LSTM
from keras.optimizers import Adam
from sklearn.feature_selection import RFE
from sklearn.linear_model import LogisticRegression
from math import sqrt
from numpy import concatenate
from sklearn.metrics import mean_squared_error
import matplotlib.pyplot as plt
import numpy

np.seterr(divide='ignore', invalid='ignore')

def loadData(riverId):
    cursor = db_config.cnx.cursor()
    sql = 'SELECT data FROM training_data WHERE river_id = %s'
    cursor.execute(sql, (riverId))
    return pickle.loads(cursor.fetchone()['data'])

def train(riverId, configId=1, data=None):
    config = getConfig(configId)

    if data == None:
        data = loadData(riverId)
    print(len(data[0]))
    data_X = data[0]
    data_y = data[1]
    print(np.array(data_X).shape)
    rng_state = np.random.get_state()
    np.random.shuffle(data_X)
    np.random.set_state(rng_state)
    np.random.shuffle(data_y)

    split = int(len(data_X) * 0.8)
    train_X = np.array(data_X[:split])
    train_y = np.array(data_y[:split])

    test_X = np.array(data_X[split:])
    test_y = np.array(data_y[split:])

    # design network
    model = Sequential()
    print(train_X.shape)
    model.add(LSTM(200, input_shape=(train_X.shape[1], train_X.shape[2])))
    model.add(Dense(1))
    model.compile(loss='mae', optimizer='adam', metrics=['mae', 'acc'])

    # fit network
    history = model.fit(train_X, train_y, epochs=500, batch_size=72, validation_data=(test_X, test_y), verbose=1, shuffle=True)
    # plot history
    # plot history
    pyplot.plot(history.history['loss'], label='train')
    pyplot.plot(history.history['val_loss'], label='test')
    pyplot.legend()
    pyplot.show()
    # make a prediction
    yhat = model.predict(test_X)

    robModel = RobModel(riverId, configId).set_model(model).save()
    return robModel

def getConfig(id):
    cursor = db_config.cnx.cursor()
    sql = 'SELECT * FROM model_configs WHERE id = %s'
    cursor.execute(sql, (id))
    return cursor.fetchone()

def getModel(riverId, configId=1):
    return RobModel(riverId, configId).load()