#!/usr/bin/env python
# -*- coding: UTF-8 -*-

import numpy as np
import pandas as pd
import random
from sklearn.metrics import confusion_matrix
import sys
import time
import copy
import heapq
from collections import Counter

from imblearn.over_sampling import SMOTE, RandomOverSampler
from imblearn.under_sampling import RandomUnderSampler
# np.random.seed(21)
# random.seed(21)
# # Follows algo from https://cs.nju.edu.cn/zhouzh/zhouzh.files/publication/icdm08b.pdf
# r=5

class WeightedIsolationTreeEnsemble:  #
    def __init__(self, sample_size, n_trees=10):

        self.sample_size = sample_size            # subsample size
        self.n_trees = n_trees                    # number of trees
        self.height_limit = np.log2(sample_size)  # height limit of an itree
        self.trees = []                           # construct multiple itrees, i.e., iForest

    def fit_improved(self, X:np.ndarray, random_state):
        """
        Given a 2D matrix of observations, create an ensemble of IsolationTree
        objects and store them in a list: self.trees.  Convert DataFrames to
        ndarray objects.
        param: X-        training samples, type-DataFrame
        param: improved- fit-improved flag, type:Boolean, default False
        return iForest
        """
        if isinstance(X, pd.DataFrame):  # isinstance(object, classinfo) to judge whether the type of object is classinfo.
            X, y = X.iloc[:, :-1].values,  X.iloc[:, -1].values  # DataFrame2Array
            len_x = len(X)  # number of instances, X.shape[0]
            col_x = X.shape[1]  # number of features
            self.trees = []  # initialize the iForest

            X_smote, y_smote = SMOTE(random_state=random_state).fit_sample(X, y)
            # ratio='auto',random_state=0,k_neighbors=2,m_neighbors=10,out_step=0.5,kind='regular',svm_estimator=None,n_jobs=1
            y_smote = sorted(Counter(y_smote).items())
            sample_idx_list = list(range(len(X_smote)))
            sample_synthetic_idx_list = list(range(len(X), len(X_smote)))

            # adjust the number of witrees
            if int(len(X_smote) /self.sample_size + 1) > self.n_trees:
                self.n_trees = int(len(X_smote) /self.sample_size + 1)

            if sample_idx_list != []:
                for i in range(self.n_trees):
                    if len(sample_idx_list) < self.sample_size:
                        self.sample_size = len(sample_idx_list)

                    sample_idx = random.sample(sample_idx_list, self.sample_size)   # randomly select the sub-samples

                    if sample_idx_list != []:
                        temp_tree = WeightedIsolationTree(self.height_limit, 0).fit_improved(X_smote[sample_idx, :],
                                                                                sample_idx, sample_synthetic_idx_list)
                        self.trees.append(temp_tree)

                    for i in sample_idx:
                        sample_idx_list.remove(i)

                return self

    def path_length_improved(self, X:np.ndarray) -> np.ndarray:
        """
        Given a 2D matrix of observations, X, compute the average path length
        for each observation in X.  Compute the path length for x_i using every
        tree in self.trees then compute the average for each x_i.  Return an
        ndarray of shape (len(X),1).
        """
        pl_vector = []                    # initialize path lengths of X
        if isinstance(X, pd.DataFrame):   # type of X is pd.DataFrame
            # X, y = X.iloc[:, :-1], X.iloc[:, -1]
            X = X.values  # DataFrame2Array
            # y = y.values
        for x in (X):

            pl = np.array([path_length_witree(x, t, 0) for t in self.trees])  # x: an instance in X, t: an iTree in iForest, e=0 initialize the current length
            # print(pl)
            pl = pl.mean()

            pl_vector.append(pl)

        pl_vector = np.array(pl_vector).reshape(-1, 1)

        return pl_vector

    def anomaly_score(self, X:np.ndarray) -> np.ndarray:
        """
        Given a 2D matrix of observations, X, compute the anomaly score
        for each x_i observation, returning an ndarray of them.
        """
        return 2.0 ** (-1.0 * self.path_length_improved(X) / c(len(X)))

    def predict_from_anomaly_scores(self, scores:np.ndarray, threshold:float) -> np.ndarray:
        """
        Given an array of scores and a score threshold, return an array of
        the predictions: 1 for any score >= the threshold and 0 otherwise.
        """

        predictions = [1 if p[0] >= threshold else 0 for p in scores]

        return predictions

    def predict_threshold(self, X:np.ndarray, threshold: float) -> np.ndarray:
        "A shorthand for calling anomaly_score() and predict_from_anomaly_scores()."

        scores = 2.0 ** (-1.0 * self.path_length_improved(X) / c(len(X)))

        predictions = [1 if p[0] >= threshold else 0 for p in scores]

        return predictions

    def predict_percentile(self, X: np.ndarray, percentile: float) -> np.ndarray:
        "Calling anomaly_score() and predict anomaly points by percentile."

        scores = 2.0 ** (-1.0 * self.path_length_improved(X) / c(len(X)))

        topk = int(len(X) * percentile)  # the number of the max value index

        maxscores_list, max_num_index = selectTopkNumbers(scores, topk)

        predictions = [1 if p in max_num_index else 0 for p in range(len(scores))]

        return predictions

class WeightedIsolationTree:

    def __init__(self, height_limit, current_height):
        self.height_limit = height_limit       # height limit of weighted-iTree
        self.current_height = current_height   # current height of an instance x
        self.split_by = None                   # split feature index
        self.split_value = None                # split value of the feature
        self.right = None                      # right leaf
        self.left = None                       # left leaf
        self.size = 0                          # number of instance in a weighted-iTree
        self.exnodes = 0                       # external nodes, initialize to 0
        self.n_nodes = 1                       # number of nodes from this layer to the bottom layer, initialize to 1
        self.left_idx = None                   # index of samples on the left node
        self.right_idx = None                  # index of samples on the right node
        self.w = 0                             # the weight of each node

    def fit_improved(self, X: np.ndarray, X_idx, sample_synthetic_idx_list):
        """
        Add Extra while loop
        """
        n_sample_synthetic = 0
        for i in X_idx:
            if i in sample_synthetic_idx_list:
                n_sample_synthetic += 1
        # if len(X) == 0:
        #     self.w = 1
        # else:
        #     self.w = (len(X) - n_sample_synthetic) / len(X)
        # print("lenx", len(X))
        self.w = (len(X) - n_sample_synthetic) / len(X)

        if len(X) <= 1 or self.current_height >= self.height_limit:
            self.exnodes = 1
            self.size = len(X)
            return self

        split_by = random.choice(np.arange(X.shape[1]))
        min_x = X[:, split_by].min()
        max_x = X[:, split_by].max()

        if min_x == max_x:
            self.exnodes = 1
            self.size = len(X)
            return self

        # split_value = min_x + random.betavariate(0.5, 0.5) * (max_x - min_x)
        split_value = min_x + random.random() * (max_x - min_x)
        a = X[X[:, split_by] < split_value]
        b = X[X[:, split_by] >= split_value]
        tf = np.where(X[:, split_by] < split_value, True, False)  # np.where(condition, x, y), condition->x, not condition->y
        # del X[:, split_by]  # delete the selected feature column
        temp_left_idx = [i for i, tfi in enumerate(list(tf)) if tfi]  # list(tf).index(True)
        temp_right_idx = [i for i, tfi in enumerate(list(tf)) if ~tfi]  # list(tf).index(False)

        self.size = X.shape[0]
        self.split_by = split_by
        self.split_value = split_value

        self.left_idx = []
        for j in temp_left_idx:
            self.left_idx.append(X_idx[j])

        self.right_idx = []
        for j in temp_right_idx:
            self.right_idx.append(X_idx[j])

        self.left = WeightedIsolationTree(self.height_limit, self.current_height + self.w*1).fit_improved(a, self.left_idx, sample_synthetic_idx_list)
        self.right = WeightedIsolationTree(self.height_limit, self.current_height + self.w*1).fit_improved(b, self.right_idx, sample_synthetic_idx_list)
        self.n_nodes = self.left.n_nodes + self.right.n_nodes + 1

        return self

def c(n):
    if n > 2:
        return 2.0*(np.log(n-1)+0.5772156649) - (2.0*(n-1.)/(n*1.0))
    elif n == 2:
        return 1
    if n == 1:
        return 0
    # if n == 0:   # why n==0 occurs?
    #     return 0

def path_length_witree(x, t, e):

    e = e
    w = t.w
    if t.exnodes == 1:
        e = e + w * c(t.size)
        return e
    else:
        a = t.split_by
        if x[a] < t.split_value:
            # n_l = 0
            # for i in x_synthetic_idx:
            #     if i in t.left_idx:
            #         n_l += 1
            # w_l = (len(t.left_idx) - n_l)/len(t.left_idx)
            return path_length_witree(x, t.left, e + w * 1)

        if x[a] >= t.split_value:
            # n_r = 0
            # for i in x_synthetic_idx:
            #     if i in t.right_idx:
            #         n_r += 1
            # w_r = (len(t.right_idx) - n_r)/len(t.right_idx)
            return path_length_witree(x, t.right, e + w * 1)

def detect_anomalies_improved(X:pd.DataFrame, random_state, sample_size=256, n_trees = 100, threshold=0.5, percentile=0.1):

    N = len(X)
    wit = WeightedIsolationTreeEnsemble(sample_size=sample_size, n_trees=n_trees)
    fit_start = time.time()
    wit.fit_improved(X, random_state=random_state)    # note: input is original samples: X
    fit_stop = time.time()
    fit_time = fit_stop - fit_start
    print(f"fit time {fit_time:3.2f}s")
    if isinstance(X, pd.DataFrame):
        X_real, y_real = X.iloc[:, :-1].values, X.iloc[:, -1].values
        # print("X_real, y_real:", len(X_real),  len(y_real), '\n', y_real)
        X_smote, y_smote = SMOTE(random_state=random_state).fit_sample(X_real, y_real)
        # ratio='auto',random_state=0,k_neighbors=2,m_neighbors=10,out_step=0.5,kind='regular',svm_estimator=None,n_jobs=1
        y_smote_class = sorted(Counter(y_smote).items())
        # print("X_smote, y_smote:", len(X_smote), len(y_smote), '\n', y_smote)
        pathlength_start = time.time()
        pathlengths = wit.path_length_improved(X_smote)   # note: input is smoted samples: X_smote
        # print("path lengths:\n", pathlengths, len(pathlengths))
        pathlength_stop = time.time()
        pathlength_time = pathlength_stop - pathlength_start
        # print(f"path length time {pathlength_time:3.2f}s")

        score_start = time.time()
        scores = wit.anomaly_score(X_smote)   # note: input is smoted samples: X_smote
        # print("anomaly scores:\n", scores, len(scores))
        score_stop = time.time()
        score_time = score_stop - score_start
        # print(f"score time {score_time:3.2f}s")

        y_predict1 = wit.predict_threshold(X_smote, threshold)     # note: input is smoted samples: X_smote
        y_predict2 = wit.predict_percentile(X_smote, percentile)   # note: input is smoted samples: X_smote
        # print(y_predict1, '\n', y_predict2)

        leave_X, leave_y = [], []
        left_idx = []
        for i in range(len(y_predict2)):
            if y_predict2[i] == 0:
                leave_X.append(X_smote[i])
                leave_y.append(y_smote[i])
                left_idx.append(i)

        leave_X = np.array(leave_X)
        leave_y = np.array(leave_y)
        # leftData = np.c_[leave_X, leave_y]
        # print(left_idx, '\n',  leave_y, '\n', leftData)

        return left_idx, leave_X, leave_y

def selectTopkNumbers(scores, topk):
    '''

    param: scores
    param: topk
    '''

    tmp_scores_list = copy.deepcopy(list(scores))  # deepcopy list: scores to avoid changing scores
    sorted_scores_list = []
    max_num_index = []
    for i in range(len(tmp_scores_list)):
        index_i = tmp_scores_list.index(max(tmp_scores_list))  # get the max value and the index of 'scores'
        max_num_index.append(index_i)  # append the index in 'max_num_index'
        tmp_scores_list[index_i] = float('-inf')  # set "-inf" when scanning the max value of scores to avoid selecting it again
        sorted_scores_list.append(scores[max_num_index[i]])
    # print("the index from the max to the min:", max_num_index)
    # print("the values from the max to the min:", sorted_scores_list)

    selected_scores_list = []
    # print("topk:", topk)

    if topk < len(sorted_scores_list):
        temp_value = sorted_scores_list[topk-1]     # the last max value
        if temp_value == sorted_scores_list[topk]:  # the last max has many equal values
            temp_idex = []  # temp index of sorted_scores_list
            n = sorted_scores_list.index(temp_value)  # the first location of the last max value
            for s in range(len(sorted_scores_list)):
                if sorted_scores_list[s] == temp_value:
                    temp_idex.append(max_num_index[s])

            # print(temp_idex, n)   # = selected_index+
            selected_index = max_num_index[:n] + random.sample(temp_idex, topk-n)
            for l in selected_index:
                selected_scores_list.append(scores[l])
        else:
            selected_scores_list = sorted_scores_list[: topk]
            selected_index = max_num_index[: topk]
    else:
        selected_scores_list = sorted_scores_list
        selected_index = max_num_index
    return selected_scores_list, selected_index
    # print("scores:", selected_scores_list, '\n', "original_scores_idx:", selected_index)

if __name__ == '__main__':
    np.random.seed(21)
    random.seed(21)
    r = 5
    # print(c(0), c(1), c(2), c(3), '\n',  c(4), c(5), c(9))
    datafile = 'cancer.csv'
    df = pd.read_csv(datafile)
    targetcol = 'diagnosis'
    sample_size = int(16)
    n_trees = int(2)

    allinstancesFlag = True
    subset_size = 32
    if allinstancesFlag:
        N = len(df)
    else:
        N = int(subset_size)

    # df = df.sample(N)  # grab random subset (too slow otherwise)
    df = df.iloc[103:135, :]
    DATA = df
    # print("InputData:", DATA)
    # X, y = df.drop(targetcol, axis=1), df[targetcol]
    left_idx, leftX, lefty = detect_anomalies_improved(DATA, random_state=r + 1, sample_size=sample_size, n_trees=n_trees,  threshold=0.65, percentile=0.1)
    print("wif:", left_idx, leftX, lefty, len(lefty))


#   Create a Python file called "amazon.py" in this folder.
#   Write code to create user-defined functions for the following operations: Minimum, Maximum, Sum, Average, and Standard Deviation.
#   These functions should run on a defined list, and write the outputs to a text file called output.txt.
#   Define a list of values as follows:
#   list = [1, 5, 8, 77, 24, 95] (you can make this list as long as you want, just remember the commas).
#   Your program should generate output.txt as follows:
#   The min of [1, 5, 8, 77, 24, 95]  is 1.
#   The max of  [1, 5, 8, 77, 24, 95]  is 95.
#   The avg of  [1, 5, 8, 77, 24, 95]  is 35.
#   Your program should handle any length of input numbers. You can assume that the list of input numbers are always valid integers and the list is never empty.

import math

list = [1, 5, 8, 77, 24, 95]

def sumAll(n):
    return sum(n)

def sumAll2(n):
    x=0
    for i in n:
        x += i
    return x

def aveAll(n):
    return sum(n)/len(n)

def aveAll2(n):
    x=0
    for i in n:
        x += i
    x/=len(n)
    return x

def maximum(n):
    return max(n)

def maximum2(n):
    x=0
    for i in n:
        if i > x:
            x=i
    return x

def minimum(n):
    return min(n)

def minimum2(n):
    x= maximum2(n)
    for i in n:
        if i < x:
            x=i
    return x

def deviation(n):
    variance=0
    for i in n:
        variance+=(i-aveAll(n))**2
    variance/=len(n)
    return math.sqrt(variance)

output = file("output.txt","w")
output = open("output.txt","r+")
output.write("The sum of " + str(list) + " is " + str(sumAll(list)) + "\n")
output.write("The average of " + str(list) + " is " + str(aveAll(list)) + "\n")
output.write("The minimum of " + str(list) + " is " + str(minimum(list)) + "\n")
output.write("The maximum of " + str(list) + " is " + str(maximum(list)) + "\n")
output.write("The standard deviation of " + str(list) + " is " + str(deviation(list)) + "\n")
output.close()

print("writting the following to output.txt")
print("The sum of " + str(list) + " is " + str(sumAll(list)))
print("The average of " + str(list) + " is " + str(aveAll(list)))
print("The minimum of " + str(list) + " is " + str(minimum(list)))
print("The maximum of " + str(list) + " is " + str(maximum(list)))
print("The standard deviation of " + str(list) + " is " + str(deviation(list)))


#!/usr/bin/python

import argparse
prices = [100, 90, 80, 50, 20, 10]
def find_max_profit(prices):
  high_profit = 0
  for i in range(1, len(prices) ):
    current_price = i
    profit = 0
    for j in range(0, current_price):
      profit = prices[i] - prices[j]
      if high_profit == 0:
        high_profit = profit
      if profit > high_profit:
        high_profit = profit

  return high_profit

print(find_max_profit(prices))

if __name__ == '__main__':
  # This is just some code to accept inputs from the command line
  parser = argparse.ArgumentParser(description='Find max profit from prices.')
  parser.add_argument('integers', metavar='N', type=int, nargs='+', help='an integer price')
  args = parser.parse_args()

  print("A profit of ${profit} can be made from the stock prices {prices}.".format(profit=find_max_profit(args.integers), prices=args.integers))


#These methods are based on Garrelt's IDL routines

from .. import const
from .. import conv
import numpy as np
import os
from .. import files
from temperature import calc_dt
from .. import utils

def freq_axis(z_low, output_slices, box_length_slices=256, box_length_mpc = conv.LB):
	''' Make a frequency axis vector with equal spacing in co-moving LOS coordinates.
	z_low is the lowest redshift and box_length is the desired number of datapoints on the axis
	Return a tuple with the redshifts and corresponding 21 cm line frequencies'''

	#TODO: add option to input z_high

	z = z_low
	z_array = np.zeros((output_slices,2))

	for nx in xrange(output_slices):
		nu = const.nu0/(1.0+z)

		z_array[nx,0] = z
		z_array[nx,1] = nu

		dnu = const.nu0*const.Hz(z)*box_length_mpc/(1.0 + z)**2/const.c/float(box_length_slices)

		z = const.nu0/(nu - dnu) - 1.0

	return z_array[:,0], z_array[:,1]

def freq_box(xfrac_dir, dens_dir, z_low, cube_slices=100):
	''' Make frequency boxes of density, ionized fractions, and brightness temperature
	-- xfrac_dir: directory containing xfrac files
	-- dens_dir: directory containing density files
	-- z_low: lowest redshift to include
	-- cube_slices: number of slices to divide the cube into

	Return tuple with density box , xfrac box, dt box , redshifts '''

	#Get the list of redshifts where we have simulation output files
	dens_redshifts = utils.get_dens_redshifts(dens_dir, z_low )

	#Get the list of redhifts and frequencies that we want for the observational box
	output_z, output_freq = freq_axis(z_low, cube_slices)
	output_z = np.delete(output_z, np.where(output_z < dens_redshifts[0])[0])
	output_z = np.delete(output_z, np.where(output_z > dens_redshifts[-1])[0])

	utils.print_msg( 'Number of slices reduced to: %d' % len(output_z) )

	#Keep track of output simulation files to use
	xfrac_filename_high = None; xfrac_filename_low = None
	xfrac_file_low = files.XfracFile(); xfrac_file_high = files.XfracFile()
	dens_filename_high = None; dens_filename_low = None
	dens_file_low = files.DensityFile(); dens_file_high = files.DensityFile()
	z_bracket_low = None; z_bracket_high = None

	#The current position in comoving coordinates
	nx = 0

	#Build the cube
	xfrac_cube = None
	dens_cube = None
	dt_cube = None
	for z in output_z:
		utils.print_msg('z=%.3f' % z)
		#Find the output files that bracket the redshift
		z_bracket_low_new = dens_redshifts[np.where(dens_redshifts <= z)[0][0]]
		z_bracket_high_new = dens_redshifts[np.where(dens_redshifts >= z)[0][0]]

		if z_bracket_low_new != z_bracket_low:
			z_bracket_low = z_bracket_low_new
			xfrac_file_low = files.XfracFile(os.path.join(xfrac_dir, 'xfrac3d_%.3f.bin' % z_bracket_low))
			dens_file_low = files.DensityFile(os.path.join(dens_dir, '%.3fn_all.dat' % z_bracket_low))
			dt_cube_low = calc_dt(xfrac_file_low, dens_file_low)

		if z_bracket_high_new != z_bracket_high:
			z_bracket_high = z_bracket_high_new
			xfrac_file_high = files.XfracFile(os.path.join(xfrac_dir, 'xfrac3d_%.3f.bin' % z_bracket_high))
			dens_file_high = files.DensityFile(os.path.join(dens_dir, '%.3fn_all.dat' % z_bracket_high))
			dt_cube_high = calc_dt(xfrac_file_high, dens_file_high)

		slice_ind = nx % xfrac_file_high.mesh_x
		#The ionized fraction
		XL = xfrac_file_low.xi[slice_ind,:,:]
		XH = xfrac_file_high.xi[slice_ind,:,:]
		Xz = ((z-z_bracket_low)*XH + (z_bracket_high - z)*XL)/(z_bracket_high-z_bracket_low) #Interpolate between slices
		if xfrac_cube == None:
			xfrac_cube = np.zeros((XL.shape[0], XL.shape[1], len(output_z)))
		xfrac_cube[:,:,nx] = Xz

		#The density
		rho_L = dens_file_low.cgs_density[slice_ind,:,:]
		rho_H = dens_file_high.cgs_density[slice_ind,:,:]
		rho_Z = ((z-z_bracket_low)*rho_H + (z_bracket_high - z)*rho_L)/(z_bracket_high-z_bracket_low) #Interpolate between slices
		if dens_cube == None:
			dens_cube = np.zeros((XL.shape[0], XL.shape[1], len(output_z)))
		dens_cube[:,:,nx] = rho_Z

		#The brightness temperature
		dt_L = dt_cube_low[slice_ind,:,:]
		dt_H = dt_cube_high[slice_ind,:,:]
		dt_Z = ((z-z_bracket_low)*dt_H + (z_bracket_high - z)*dt_L)/(z_bracket_high-z_bracket_low) #Interpolate between slices
		if dt_cube == None:
			dt_cube = np.zeros((XL.shape[0], XL.shape[1], len(output_z)))
		dt_cube[:,:,nx] = dt_Z

		utils.print_msg( 'Slice %d of %d' % (nx, len(output_z)) )
		nx += 1

	return xfrac_cube, dens_cube, dt_cube, output_z


# Copyright 2017 reinforce.io. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================

"""
Test for examples from the reinforce.io website, blogposts and other examples.
"""

from __future__ import absolute_import
from __future__ import print_function
from __future__ import division

import unittest

class TestTutorialCode(unittest.TestCase):
    """
    Validation of random code snippets as to be notified when old blog posts need to be changed.
    """

    class MyClient(object):

        def __init__(self, *args, **kwargs):
            pass

        def get_state(self):
            import numpy as np
            return np.random.rand(10)

        def execute(self, action):
            pass

    def test_reinforceio_homepage(self):
        """
        Code example from the homepage and README.md.
        """

        from tensorforce import Configuration
        from tensorforce.agents import TRPOAgent
        from tensorforce.core.networks import layered_network_builder

        config = Configuration(
            batch_size=100,
            states=dict(shape=(10,), type='float'),
            actions=dict(continuous=False, num_actions=2),
            network=layered_network_builder([dict(type='dense', size=50), dict(type='dense', size=50)])
        )

        # Create a Trust Region Policy Optimization agent
        agent = TRPOAgent(config=config)

        # Get new data from somewhere, e.g. a client to a web app
        client = TestTutorialCode.MyClient('http://127.0.0.1', 8080)

        # Poll new state from client
        state = client.get_state()

        # Get prediction from agent, execute
        action = agent.act(state=state)
        reward = client.execute(action)

        # Add experience, agent automatically updates model according to batch size
        agent.observe(reward=reward, terminal=False)

    def test_blogpost_introduction(self):
        """
        Test of introduction blog post examples.
        """
        import tensorflow as tf
        import numpy as np

        ### DQN agent example

        from tensorforce import Configuration
        from tensorforce.agents import DQNAgent
        from tensorforce.core.networks import layered_network_builder

        # Define a network builder from an ordered list of layers
        layers = [dict(type='dense', size=32),
                  dict(type='dense', size=32)]
        network = layered_network_builder(layers_config=layers)

        # Define a state
        states = dict(shape=(10,), type='float')

        # Define an action (models internally assert whether
        # they support continuous and/or discrete control)
        actions = dict(continuous=False, num_actions=5)

        # The agent is configured with a single configuration object
        agent_config = Configuration(
            batch_size=8,
            learning_rate=0.001,
            memory_capacity=800,
            first_update=80,
            repeat_update=4,
            target_update_frequency=20,
            states=states,
            actions=actions,
            network=network
        )
        agent = DQNAgent(config=agent_config)

        ### Code block: multiple states

        states = dict(
            image=dict(shape=(64, 64, 3), type='float'),
            caption=dict(shape=(20,), type='int')
        )

        agent_config.states = states
        # DQN does not support multiple states. Omit test for now.
        # agent = DQNAgent(config=agent_config)

        ### Code block: DQN observer function

        def observe(self, reward, terminal):
            super(DQNAgent, self).observe(reward, terminal)
            if self.timestep >= self.first_update \
                    and self.timestep % self.target_update_frequency == 0:
                self.model.update_target()

        ### Code block: Network config JSON

        network_json = """
        [
            {
                "type": "conv2d",
                "size": 32,
                "window": 8,
                "stride": 4
            },
            {
                "type": "conv2d",
                "size": 64,
                "window": 4,
                "stride": 2
            },
            {
                "type": "flatten"
            },
            {
                "type": "dense",
                "size": 512
            }
        ]
        """

        ### Test json

        import json
        network_config = json.loads(network_json)
        network = layered_network_builder(network_config)

        ### Test from_json import

        from tensorforce.core.networks import from_json

        ### Code block: Modified dense layer

        modified_dense = """
        [
            {
                "type": "dense",
                "size": 64,
                "bias": false,
                "activation": "selu",
                "l2_regularization": 0.001
            }
        ]
        """

        ### Test json

        network_config = json.loads(network_json)
        network = layered_network_builder(network_config)

        ### Code block: Own layer type

        def batch_normalization(x, variance_epsilon=1e-6, **kwargs):
            mean, variance = tf.nn.moments(x, axes=tuple(range(x.shape.ndims - 1)))
            x = tf.nn.batch_normalization(x, mean=mean, variance=variance, offset=None, scale=None,
                                          variance_epsilon=variance_epsilon)
            return x

        ### Test own layer

        network_config = [{"type": batch_normalization,
                           "variance_epsilon": 1e-9}]
        network = layered_network_builder(network_config)
        agent_config.network=network
        agent_config.states = dict(shape=(10,), type='float')
        agent = DQNAgent(config=agent_config)

        ### Code block: Own network builder

        def network_builder(inputs, **kwargs):
            image = inputs['image']  # 64x64x3-dim, float
            caption = inputs['caption']  # 20-dim, int

            with tf.variable_scope('cnn'):
                weights = tf.Variable(tf.random_normal(shape=(3, 3, 3, 16), stddev=0.01))
                image = tf.nn.conv2d(image, filter=weights, strides=(1, 1, 1, 1), padding='SAME')
                image = tf.nn.relu(image)
                image = tf.nn.max_pool(image, ksize=(1, 2, 2, 1), strides=(1, 2, 2, 1), padding='SAME')

                weights = tf.Variable(tf.random_normal(shape=(3, 3, 16, 32), stddev=0.01))
                image = tf.nn.conv2d(image, filter=weights, strides=(1, 1, 1, 1), padding='SAME')
                image = tf.nn.relu(image)
                image = tf.nn.max_pool(image, ksize=(1, 2, 2, 1), strides=(1, 2, 2, 1), padding='SAME')

                image = tf.reshape(image, shape=(-1, 16 * 16, 32))
                image = tf.reduce_mean(image, axis=1)

            with tf.variable_scope('lstm'):
                weights = tf.Variable(tf.random_normal(shape=(30, 32), stddev=0.01))
                caption = tf.nn.embedding_lookup(params=weights, ids=caption)
                lstm = tf.contrib.rnn.LSTMCell(num_units=32)
                caption, _ = tf.nn.dynamic_rnn(cell=lstm, inputs=caption, dtype=tf.float32)
                caption = tf.reduce_mean(caption, axis=1)

            return tf.multiply(image, caption)

        ### Test own network builder

        agent_config.states = dict(
            image=dict(shape=(64, 64, 3), type='float'),
            caption=dict(shape=(20,), type='int')
        )
        agent_config.network=network_builder
        agent = DQNAgent(config=agent_config)

        ### Code block: LSTM function

        def lstm(x, **kwargs):
            size = x.get_shape()[1].value
            internal_input = tf.placeholder(dtype=tf.float32, shape=(None, 2, size))
            lstm = tf.contrib.rnn.LSTMCell(num_units=size)
            state = tf.contrib.rnn.LSTMStateTuple(internal_input[:, 0, :],
                                                  internal_input[:, 1, :])
            x, state = lstm(inputs=x, state=state)
            internal_output = tf.stack(values=(state.c, state.h), axis=1)
            internal_init = np.zeros(shape=(2, size))
            return x, [internal_input], [internal_output], [internal_init]

        ### Test LSTM

        network_config = [{"type": lstm}]
        network = layered_network_builder(network_config)
        agent_config.network=network
        agent_config.states = dict(shape=(10,), type='float')
        agent = DQNAgent(config=agent_config)

        ### Preprocessing configuration

        agent_config.preprocessing = [
            dict(
                type='image_resize',
                width=84,
                height=84
            ),
            dict(
                type='grayscale'
            ),
            dict(
                type='center'
            ),
            dict(
                type='sequence',
                length=4
            )
        ]

        ### Test preprocessing configuration

        agent = DQNAgent(config=agent_config)

        #agent_config.actions = dict(continuous=False, num_actions=5)

        ### Code block: Continuous action exploration

        agent_config.exploration = dict(
            type='OrnsteinUhlenbeckProcess',
            sigma=0.1,
            mu=0,
            theta=0.1
        )

        ### Test continuous action exploration

        agent = DQNAgent(config=agent_config)

        ### Code block: Discrete action exploration

        agent_config.exploration = dict(
            type='EpsilonDecay',
            epsilon=1,
            epsilon_final=0.01,
            epsilon_timesteps=1e6
        )

        ### Test discrete action exploration

        agent = DQNAgent(config=agent_config)

    def test_blogpost_introduction_runner(self):
        from tensorforce.config import Configuration
        from tensorforce.core.networks import layered_network_builder
        from tensorforce.environments.minimal_test import MinimalTest
        from tensorforce.agents import DQNAgent
        from tensorforce.execution import Runner

        environment = MinimalTest(definition=False)

        network_config = [
            dict(type='dense', size=32)
        ]
        agent_config = Configuration(
            batch_size=8,
            learning_rate=0.001,
            memory_capacity=800,
            first_update=80,
            repeat_update=4,
            target_update_frequency=20,
            states=environment.states,
            actions=environment.actions,
            network=layered_network_builder(network_config)
        )

        agent = DQNAgent(config=agent_config)
        runner = Runner(agent=agent, environment=environment)

        def episode_finished(runner):
            if runner.episode % 100 == 0:
                print(sum(runner.episode_rewards[-100:]) / 100)
            return runner.episode < 100 \
                   or not all(reward >= 1.0 for reward in runner.episode_rewards[-100:])

        # runner.run(episodes=1000, episode_finished=episode_finished)
        runner.run(episodes=10, episode_finished=episode_finished)  # Only 10 episodes for this test

        ### Code block: next

        # max_episodes = 1000
        max_episodes = 10  # Only 10 episodes for this test
        max_timesteps = 2000

        episode = 0
        episode_rewards = list()

        while True:
            state = environment.reset()
            agent.reset()

            timestep = 0
            episode_reward = 0
            while True:
                action = agent.act(state=state)
                state, reward, terminal = environment.execute(action=action)
                agent.observe(reward=reward, terminal=terminal)

                timestep += 1
                episode_reward += reward

                if terminal or timestep == max_timesteps:
                    break

            episode += 1
            episode_rewards.append(episode_reward)

            if all(reward >= 1.0 for reward in episode_rewards[-100:]) \
                    or episode == max_episodes:
                break


"""returns value if present in list of given valid values.
	- if list of dicts, a key can be given to do a key:value check and return the valid dict.
	- a default value can be given to be returned in case of no match.
"""
def check_return_parameter(check_value, param_list, param_key=None, default=None):

	# check list type (dict or normal)
	list_type_dict = True if isinstance(param_list[0], dict) else False

	if(list_type_dict):
		for x in param_list:
			if(x[param_key] == check_value): return x

		if(default != None):
			try: return list(filter(lambda x: x[param_key] == default, param_list))[0]
			except: raise AttributeError("default value given is not present in list")

		return False

	else:
		if(check_value in param_list):
			return check_value

		if(default != None):
			if(default in param_list):
				return default

			raise AttributeError("default value given is not present in list")

		return False

# -- example data and use cases ------------------------------------------------------------

datums = [
	{
		"code": "msl",
		"name": "Mean Sea Level",
	},
	{
		"code": "lat",
		"name": "Lowest Astronomical Tide",
	},
	{
		"code": "man",
		"name": "Manual",
	},
	{
		"code": "chart",
		"name": "Chart",
	},
	{
		"code": "ord",
		"name": "Ordnance",
	},
]

time_intervals = [5, 6, 10, 12, 15, 20, 30, 60]

check_return_parameter(15, time_intervals) 					# return 15
check_return_parameter(24, time_intervals) 					# returns False
check_return_parameter(24, time_intervals, default=15)		# returns 15

check_return_parameter("lat", datums, param_key="code")					# returns { "code": "lat", "name": "Lowest Astronomical Tide" }
check_return_parameter("abc", datums, param_key="code") 				# returns false
check_return_parameter("abc", datums, param_key="code", default="msl")	# returns { "code": "msl", "name": "Mean Sea Level" }
check_return_parameter("abc", datums, param_key="code", default="def")	# returns 'AttributeError: default value given is not present in list'


import numpy as np
from pyzbar import pyzbar
from threading import Thread
import cv2
import motors as mot
import target_process as tp
import log as log

last_qr_text = None

def detect_qrcode(img, show):
	"""
	Detect qrcodes from the camera
	:param img: Image analysed
	:param show: Boolean, do we want to see what's happening?
	:return: The text from the qrcode
	"""
	global last_qr_text
	codes = pyzbar.decode(img)

	try:
		if len(codes) > 0:
			for code in codes:
				qr_text = code.data.decode("utf-8")

				if qr_text != last_qr_text and tp.get_target() >= 0:
					get_node_direction(qr_text, tp.get_target())
					log.debug("QRCODE " + qr_text)
					last_qr_text = qr_text

				if show:
					points = code.polygon

					pt1 = (min(points[0][0], points[2][0]), min(points[0][1], points[2][1]))
					pt2 = (max(points[0][0], points[2][0]), max(points[0][1], points[2][1]))

					cv2.rectangle(img, pt1, pt2, (230, 230, 230), 2)
					# pt1 en haut a gauche
					# pt2 en bas a droite

					cv2.putText(img, qr_text, (pt1[0], pt1[1] - 20), cv2.FONT_HERSHEY_SIMPLEX, 0.5, (230, 230, 230), 2)

				return qr_text

	except IndexError:
		log.debug("IndexError from detect_qrcode")

def get_node_direction(qrtext, target):
	"""
	From the text and the table number, get the direction and call mot.do_turn with it.
	:param qrtext: Text from the qrcode
	:param target: The table number
	"""
	if qrtext == "TARGET":
		t = Thread(target=mot.elevation())
		t.start()
	else:
		data = qrtext.split("/")

		for d in data:
			data_bis = d.split()

			try:
				for d_bis in data_bis:
					if int(d_bis) == target:
						id_turn = data.index(d)

						if not mot.stop_call and not mot.sensor_collision:
							log.debug("ID:" + str(id_turn))
							t = Thread(target=mot.do_turn, args=[201 + id_turn])
							t.start()
						break

			except ValueError:
				log.debug("ValueError from get_node_direction")

def process_contours(img, show):
	"""
	Find the contours of black figures to get their center
	:param img: Image analysed
	:param show: Boolean, do we want to see what's happening?
	:return: The position x of the figure's center
	"""
	gris = np.array(30, dtype="uint8")
	noir = np.array(0, dtype="uint8")

	biggest_contour = None
	mask = 255 - cv2.inRange(img, noir, gris)
	ret, tresh = cv2.threshold(mask, 127, 255, cv2.THRESH_BINARY_INV)
	contours = cv2.findContours(tresh, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)[0]

	if len(contours) > 0:
		for c in contours:
			if biggest_contour is None:
				biggest_contour = c

			if cv2.contourArea(c) > cv2.contourArea(biggest_contour):
				biggest_contour = c

		if show:
			cv2.drawContours(img, biggest_contour, -1, (250, 250, 250), 3)

		m = cv2.moments(biggest_contour)

		if m["m10"] != 0 and m["m00"] != 0 and m["m01"] != 0:
			c_x = int(m["m10"] / m["m00"])
			c_y = int(m["m01"] / m["m00"])

			if c_x != 219 and c_y != 119 and show:
				cv2.circle(img, (c_x, c_y), 7, (255, 255, 255), -1)

			return c_x

		else:
			return 0

	else:
		return 0

def send_order_direction(xmargin, imgwidth, cx):
	"""
	Depending of a position x on the image, create a coefficient between -100 and 100
	:param xmargin: The dead zone
	:param imgwidth: The width of the image
	:param cx: The position x on the image
	"""
	xcenter_img = imgwidth / 2

	xlimit_left = xcenter_img - (xmargin / 2)
	xlimit_right = xcenter_img + (xmargin / 2)

	# print("--------")
	# print("WIDTH " + str(imgwidth))
	# print("CENTER " + str(xcenter_img))
	# print("cx "+ str(cx))
	# print("RIGHT "+ str(xlimit_right))
	# print("LEFT "+ str(xlimit_left))

	if cx < xlimit_left:
		coeff = cx / xlimit_left - 1

	elif cx > xlimit_right:
		coeff = (cx - xlimit_right) / (imgwidth - xlimit_right)

	else:
		coeff = 0

	mot.prevent_collision()  # We have to check if nothing is blocking the robot

	if not mot.stop_call and not mot.sensor_collision:
		mot.adapt_duty_cycle_dep(int(coeff * 100))

# coeff < 0 -> Go left
# coeff > 0 -> Go right


import argparse
import boto3
import uuid

parser = argparse.ArgumentParser()
parser.add_argument('-r', '--region',
                    metavar='{REGION}', required=True)
parser.add_argument('-a', '--api_name',
                    metavar='{API_NAME}',  default='gelbeseiten')
parser.add_argument('-l', '--lambda_name',
                    metavar='{LAMBDA_NAME}',  default='gelbeseiten')
parser.add_argument('-c', '--cache_disable', action='store_true')

args = parser.parse_args()

AWS_REGION = args.region
LAMBDA_NAME = args.lambda_name
API_NAME = args.api_name
CACHE_DISABLE = args.cache_disable
API_DESC = 'gelbeseiten.de scraper API'

###############################################################################
# Init
###############################################################################

aws_gateway = boto3.client('apigateway', AWS_REGION)
aws_lambda = boto3.client('lambda', AWS_REGION)

aws_account = boto3.client('sts').get_caller_identity().get('Account')

lambda_version = aws_lambda.meta.service_model.api_version
lambda_uri_data = {
    'aws-region': AWS_REGION,
    'aws-acct-id': aws_account,
    'lambda-function-name': LAMBDA_NAME,
    'api-version': lambda_version,
}
lambda_uri = 'arn:aws:apigateway:{aws-region}:lambda:path/{api-version}/functions/arn:aws:lambda:{aws-region}:{aws-acct-id}:function:{lambda-function-name}/invocations'.format(**lambda_uri_data)

###############################################################################
# Create API
###############################################################################

api = aws_gateway.create_rest_api(
    name=API_NAME,
    description=API_DESC
)
home_resource = aws_gateway.get_resources(restApiId=api['id'])['items'][0]

###############################################################################
# Create endpoint: GET /companies
###############################################################################

companies_resource = aws_gateway.create_resource(
    restApiId=api['id'],
    parentId=home_resource['id'],
    pathPart='companies'
)

aws_gateway.put_method(
    restApiId=api['id'],
    resourceId=companies_resource['id'],
    httpMethod='GET',
    authorizationType='NONE',
    requestParameters={
        'method.request.querystring.q': True,
        'method.request.querystring.postcode': True,
    }
)

aws_gateway.put_method_response(
    restApiId=api['id'],
    resourceId=companies_resource['id'],
    httpMethod='GET',
    statusCode='200'
)

tpl = """
#set($inputRoot = $input.path('$'))
{
    "method": "companies",
    "q": "$input.params('q')",
    "postcode": "$input.params('postcode')"
}
"""

aws_gateway.put_integration(
    restApiId=api['id'],
    resourceId=companies_resource['id'],
    httpMethod='GET',
    type='AWS',
    integrationHttpMethod='POST',
    uri=lambda_uri,
    requestParameters={
        'integration.request.querystring.q': 'method.request.querystring.q',
        'integration.request.querystring.postcode': 'method.request.querystring.postcode',
    },
    requestTemplates={
        'application/json': tpl
    },
    cacheKeyParameters=[
        'method.request.querystring.q',
        'method.request.querystring.postcode']
)

aws_gateway.put_integration_response(
    restApiId=api['id'],
    resourceId=companies_resource['id'],
    httpMethod='GET',
    statusCode='200',
    selectionPattern='.*'
)

uri_data = {
    'aws-region': AWS_REGION,
    'aws-acct-id': aws_account,
    'lambda-function-name': LAMBDA_NAME,
    'api-version': lambda_version,
    'aws-api-id': api['id'],
}
source_arn = "arn:aws:execute-api:{aws-region}:{aws-acct-id}:{aws-api-id}/*/GET/companies".format(**uri_data)

aws_lambda.add_permission(
    FunctionName=LAMBDA_NAME,
    StatementId=uuid.uuid4().hex,
    Action="lambda:InvokeFunction",
    Principal="apigateway.amazonaws.com",
    SourceArn=source_arn
)

###############################################################################
# Create endpoint: GET /companies/{company_id}
###############################################################################

company_resource = aws_gateway.create_resource(
    restApiId=api['id'],
    parentId=companies_resource['id'],
    pathPart='{company_id}'
)

aws_gateway.put_method(
    restApiId=api['id'],
    resourceId=company_resource['id'],
    httpMethod='GET',
    authorizationType='NONE'
)

aws_gateway.put_method_response(
    restApiId=api['id'],
    resourceId=company_resource['id'],
    httpMethod='GET',
    statusCode='200'
)

tpl = """
#set($inputRoot = $input.path('$'))
{
    "method": "company",
    "company_id": "$input.params('company_id')"
}
"""

aws_gateway.put_integration(
    restApiId=api['id'],
    resourceId=company_resource['id'],
    httpMethod='GET',
    type='AWS',
    integrationHttpMethod='POST',
    uri=lambda_uri,
    requestTemplates={
        'application/json': tpl
    }
)

aws_gateway.put_integration_response(
    restApiId=api['id'],
    resourceId=company_resource['id'],
    httpMethod='GET',
    statusCode='200',
    selectionPattern='.*'
)

uri_data = {
    'aws-region': AWS_REGION,
    'aws-acct-id': aws_account,
    'lambda-function-name': LAMBDA_NAME,
    'api-version': lambda_version,
    'aws-api-id': api['id'],
}
source_arn = "arn:aws:execute-api:{aws-region}:{aws-acct-id}:{aws-api-id}/*/GET/companies/*".format(**uri_data)

aws_lambda.add_permission(
    FunctionName=LAMBDA_NAME,
    StatementId=uuid.uuid4().hex,
    Action="lambda:InvokeFunction",
    Principal="apigateway.amazonaws.com",
    SourceArn=source_arn
)

###############################################################################
# Deploy
###############################################################################

if CACHE_DISABLE:

    aws_gateway.create_deployment(
        restApiId=api['id'],
        stageName='dev'
    )

else:

    aws_gateway.create_deployment(
        restApiId=api['id'],
        stageName='dev',
        cacheClusterEnabled=True,
        cacheClusterSize='0.5'
    )

    aws_gateway.update_stage(
        restApiId=api['id'],
        stageName='dev',
        patchOperations=[
            {
                'op': 'replace',
                'path': '/*/*/caching/enabled',
                'value': 'True'
            },
            {
                'op': 'replace',
                'path': '/*/*/caching/ttlInSeconds',
                'value': '3600'
            },
            {
                'op': 'replace',
                'path': '/*/*/caching/requireAuthorizationForCacheControl',
                'value': 'False'
            }
        ]
    )
    print('Cache creation in progress (This will take some time)')

print('API Creation successful.')
url = 'https://{0}.execute-api.eu-central-1.amazonaws.com/dev/companies'
url = url.format(api['id'])
print('Example calls:\n')
print('  $ curl -XGET "{0}?q=pizza&postcode=10111"'.format(url))
print('  $ curl -XGET "{0}/1056800802"\n'.format(url))


#!/usr/bin/env python3

# usage: python3 kafka-jmx-metric-collector.py localhost 9999 roles/jmxMonitor/files/input.txt /tmp/ kafka-dev

# This script suppose to export all kafka metric from one node and write to file
# from where either splunk like tools can read it.

from jmxquery import *
from socket import gethostname
from datetime import datetime
import json
import sys
import psutil
import threading

class KafkaJmx:
    def __init__(self,kAddr,kPort,inputFile,logDir,env):
        self.kAddr = kAddr
        self.kPort = kPort
        self.kJmxAddr = "service:jmx:rmi:///jndi/rmi://" + str(self.kAddr) + ":" + str(self.kPort) + "/jmxrmi"
        self.cTimeNow = datetime.now()
        self.jmxConnection = JMXConnection(self.kJmxAddr)
        self.inputFile = inputFile
        self.logDir = logDir
        self.env = env
        self.domainNameList = ['java.lang','kafka.controller','kafka.log',
                                'kafka.network','kafka.server','kafka.utils']

    def getMetric(self):
        with open(self.inputFile) as file:
            for query in file:
                metrics = self.jmxConnection.query([JMXQuery(query.strip())], timeout=1000000)
                for metric in metrics:
                    domainName = metric.to_query_string().split(":")[0]
                    queryName = metric.to_query_string().split(":")[1]
                    queryValue = metric.value
                    _queryDict = {
                                "@timestamp": str(self.cTimeNow),
                                "domainName": str(domainName),
                                "environment": str(self.env),
                                "queryName": str(queryName),
                                "queryValue":  queryValue
                                }
                    with open(self.logDir + domainName + ".log", 'a+') as logFile:
                       logFile.write("\n")
                       logFile.write(json.dumps(_queryDict))

    def cleanUpFiles(self):
        for domainName in self.domainNameList:
            open(self.logDir + domainName + ".log", 'w').close()

    def getStorageMetric(self):
        _sMM = psutil.disk_usage("/kafka")
        _sMetric = {
                    "@timestamp": str(self.cTimeNow),
                    "domainName": "disk",
                    "environment": self.env,
                    "totalInGB": _sMM.total // (2**30),
                    "usedInGB":  _sMM.used // (2**30),
                    "freeInGB": _sMM.free // (2**30),
                    "usedPercent": _sMM.percent
                    }

        with open(self.logDir + "disk.log", 'w') as logFile:
            logFile.write(json.dumps(_sMetric))

    def getCpuMetric(self):
        _cMetric = {
                    "@timestamp": str(self.cTimeNow),
                    "domainName": "cpu",
                    "environment": self.env,
                    "usedCpuPercent": psutil.cpu_percent()
                    }

        with open(self.logDir + "cpu.log", 'w') as logFile:
            logFile.write(json.dumps(_cMetric))

    def getMemoryMetric(self):
        _memStats = psutil.virtual_memory()
        _swapMemStats = psutil.swap_memory()
        _rMetric = {
                    "@timestamp": str(self.cTimeNow),
                    "domainName": "memory",
                    "environment": self.env,
                    "totalMem": _memStats.total // (2**30),
                    "availableMem": _memStats.available // (2**30),
                    "percentUsedMem": _memStats.percent,
                    "usedMem": _memStats.used // (2**30),
                    "buffers": _memStats.buffers // (2**30),
                    "totalSwap": _swapMemStats.total // (2**30),
                    "usedSwap": _swapMemStats.used // (2**30),
                    "freeSwap": _swapMemStats.free // (2**30),
                    "percentUsedSwap": _swapMemStats.percent
                    }

        with open(self.logDir + "memory.log", 'w') as logFile:
            logFile.write(json.dumps(_rMetric))
def main():
    hostname = sys.argv[1]
    port = sys.argv[2]
    inputFile = sys.argv[3]
    logDir = sys.argv[4]
    env = sys.argv[5]

    z = KafkaJmx(hostname, port, inputFile, logDir,env)
    z.cleanUpFiles()

    _metric_thread = threading.Thread(
        target=z.getMetric
    ).start()

    _cpu_metric_thread = threading.Thread(
        target=z.getCpuMetric
    ).start()

    _memory_metric_thread = threading.Thread(
        target=z.getMemoryMetric
    ).start()

    _storage_metric_thread = threading.Thread(
        target=z.getStorageMetric
    ).start()

main()


import cv2
import os

outdir = 'data/sample_drive/paper/'
cap = cv2.VideoCapture('data/sample_drive/paper.avi')

try:
    if not os.path.exists(outdir):
        os.makedirs(outdir)
except OSError:
    print('Error: cannot create dir of output data.')

id = 0
while(True):
    ret, frame = cap.read()

    fn = outdir + str(id) +'.png'
    # print('Creating...' + fn)
    cv2.imwrite(fn, frame)

    id += 1

cap.release()


"""empty message

Revision ID: 7bc565eef37e
Revises: a9bae9b7fe2a
Create Date: 2021-02-09 15:27:55.428089

"""
from alembic import op
import sqlalchemy as sa

# revision identifiers, used by Alembic.
revision = '7bc565eef37e'
down_revision = 'a9bae9b7fe2a'
branch_labels = None
depends_on = None

def upgrade():
    # ### commands auto generated by Alembic - please adjust! ###
    op.add_column('todos', sa.Column('completeded', sa.Boolean(), nullable=False))
    # ### end Alembic commands ###

def downgrade():
    # ### commands auto generated by Alembic - please adjust! ###
    op.drop_column('todos', 'completeded')
    # ### end Alembic commands ###


# Wrong Answer
# Python 3

# this problem is still unsolved

import math

# this return all prime <178; cause sqrt(31700)
def find_all_prime():

    mylist=[0]*179; mylist[0] = 1; mylist[1] = 1
    for i in range(4, 179, 2):
        mylist[i] = 1

    # here 14 is the sqrt(179); which i use for run the loop
    for i in range(3, 14, 2):
        j = i+i
        while j < 179:
            mylist[j] = 1
            j += i
    li = []
    for i in range(179):
        if not mylist[i]:
            li.append(i)

    return li

# this returns all prime <31700; cause sqrt(10**9)
def find_prime():

    li = find_all_prime()
    mylist = [0]*31700; mylist[0]=1; mylist[1]=1
    # here (40) is the length of the li list
    for i in range(40):
        j = li[i] + li[i]
        while j < 31700:
            mylist[j] = 1
            j += li[i]

    arr=[]
    for i in range(31700):
        if not mylist[i]:
            arr.append(i)

    return arr

start, end = map(int, input().split())
length = end-start+1
prime = find_prime()
pr_len = len(prime)
arr = [0] * length

for i in range(pr_len):

    if start < prime[i]:
        break
    p = start//prime[i]
    r = start-(p*prime[i])
    if (p==1) and not r:
        r += prime[i]
    while r < length:
        arr[r] = 1
        r += prime[i]

count=0
for i in range(length-2):
    if (not arr[i]) and (not arr[i+2]):
            count += 1

print(count)


from django.shortcuts import render
from django.http import *
from .models import *
from django.template import *
from django.core.urlresolvers import reverse
import json

class showAssistanInfo(object):
    def __init__(self, assistantNumber, assistantName, assistantClass):
        self.assistantNumber = assistantNumber
        self.assistantName = assistantName
        self.assistantClass = assistantClass
    #返回助教的学号
    def get_Number(self):
        return self.assistantNumber
    #返回助教的名字
    def get_Name(self):
        return self.assistantName
    #返回助教的所带的班级
    def get_Class(self):
        return self.assistantClass

#显示助教信息
def managerAssistantInfo(request):
    try: #抛个异常判断，防止出现意外情况
        #首先判断是否是属于登录中的用户并且是否有足够的权限（这个模块只有老师可以进），否则跳转到登录页面中
        isOnlineUser = request.session.get('uid');
        isEnoughtPermission = request.session.get('permission');
        if isOnlineUser == None: #表示不是登录中的用户
            return HttpResponseRedirect('/index/');
        else:
            if isEnoughtPermission == 3:  #权限等于3（老师）的才能进入
                # 判断添加是否成功，拿取添加操作传递过来的数据
                addresult = request.GET.get('addresult');
                allnumber = request.GET.get('allnumber');
                if addresult == None:  # 这个表示是并没有执行添加操作
                    addresult = "";
                    addresultPrintf = "";
                else:
                    addresult = "总添加条数为" + str(allnumber) + ";其中成功添加" + str(addresult) + "条，添加失败" + str(int(allnumber) - int(addresult)) + "条(注：无法重复添加已存在信息或不存在的数据信息)"
                    addresultPrintf = addresult;

                # 得到删除操作后，传过来的内容
                deleteresult = request.GET.get('del');
                if deleteresult == 'success':
                    deleteresult = '删除成功!';
                else:
                    deleteresult = '';

                # 得到删除操作后，传过来的内容
                updataresult = request.GET.get('upd');
                if updataresult == 'success':
                    updataresult = '更新成功!';
                else:
                    updataresult = '';

                #得到当前登录老师的姓名和工号(session获取)
                contentOne = request.session.get('name');
                teacherUid = request.session.get('uid');
                # 得到user数据库中的对应老师对应的助教数据
                currentTeacherInfo = User.objects(uid = teacherUid);
                dictInfo = [];  #对应老师的助教学生的信息
                if len(currentTeacherInfo) > 0 : #找到对应的老师信息
                    currentTeacherInfo = User.objects(uid=teacherUid)[0];
                    belongtoClassNumber = currentTeacherInfo.teacher_class_name; #老师所带的学生的班级，返回是列表
                    content = currentTeacherInfo.assisant_info ; #得到助教的信息，返回的是字典
                    for everyInfo in content.keys():  #得到助教的学号
                        stuNumber = everyInfo ;
                        #通过学号找助教的姓名
                        currentUser = User.objects(uid = stuNumber)[0];
                        stuName = currentUser.name;
                        stuClass = currentUser.student_class_name;
                        #封装到对象里面
                        currentContext = showAssistanInfo(stuNumber,stuName,stuClass);
                        #添加到列表中，进行传送
                        dictInfo.append(currentContext);
                else:
                    dictInfo = [''];

                #（1）dbinfoDatda：对应老师所带的助教信息（2）addresult：进行添加操作的结果（3）addresultPrintf：进行添加操作打印信息的内容
                #（4）teacherinfo：当前登录老师的姓名（5）deleteResult：删除操作的结果（6）updataResult：更新操作的结果
                #(6)belongtoClassNumber:当前登录老师所带的班级
                context = {'dbinfoDatda': dictInfo, 'addresult': addresult ,'addresultPrintf':json.dumps(addresultPrintf), 'teacherinfo':contentOne,
                       'deleteResult': json.dumps(deleteresult) , 'updataResult':json.dumps(updataresult),
                        'belongtoClassNumber':belongtoClassNumber};
                return render(request, 'Csapphnu/managerassistantinfo.html', context);
            else: #没有足够权限的则显示到主界面
                return HttpResponseRedirect('/index/');
    except Exception as  errors:
        return HttpResponseRedirect('/index/');

#添加助教信息
def addInfo(request):
    #得到传送过来选择的索引值
    index = request.POST.get('tableinfoindex');
    #执行添加数据到数据库操作，返回添加的数目
    successresult , allnumber= addAssistantInfo2DB(index ,request);
    #重定向到最初的显示页面
    return HttpResponseRedirect('/assistantmanage/?addresult=' + str(successresult)+'&allnumber='+str(allnumber));

#将传送过来的数据添加到数据库中
def addAssistantInfo2DB(index , request):
    #分割传送过来的索引值，以逗号进行分割
    isSucess = 0;
    index = index[0:len(index)-1] #因为最后多了一个逗号，先将其进行截取
    indexlist = index.split(',');
    for con in indexlist:
        #获取输入框和选择框中填写的内容
        stuNumber = request.POST.get('assistantNumber'+con);
        stuName = request.POST.get('assistantName' + con);
        stuClass = request.POST.get('selectClass' + con);
       # stuClass = request.POST.get('assistantClass' + con);  //之前版本中的输入框的添加信息
        #判断要添加的数据是否已经在数据库中了(用学号为主键来判断)
        # 得到数据库中的数据，根据学号来进行判断
        #得到该老师的uid，来得到该老师的信息
        teacherUid = request.session.get('uid');
        currentTeacherInfo = User.objects(uid = teacherUid)[0];
        if len(currentTeacherInfo) > 0 : #表示找到对应的老师的信息
            #判断添加的助教是否在user表中，否则不能无中生有加助教的信息
            addAssistanStudent = User.objects(uid = stuNumber ,name = stuName ,permission = 2);
            if len(addAssistanStudent) > 0 :#表示该助教的信息符合
                #判断添加的助教是否已经存在对应的该老师的信息中
                dictAssistantInfo = currentTeacherInfo.assisant_info; #返回的是字典
                isNoFind = True;
                for everyassistant in dictAssistantInfo.keys():
                    if everyassistant == stuNumber:
                        isNoFind = False;
                        break;
                if isNoFind == True:  #表示没有重复，则进行添加操作
                    #添加对应的助教信息
                    currentTeacherInfo.assisant_info[stuNumber] = stuName;
                    #更新对应助教的个人信息
                    addAssistant = User.objects(uid = stuNumber, name=stuName)[0];
                    addAssistant.student_class_name = stuClass;
                    addAssistant.save();
                    currentTeacherInfo.save();
                    isSucess += 1;
                else:
                    continue;
            else:
                continue;
        else:
            continue;
    #返回添加的条数，一个为成功添加的数目，第二个为总数目
    return isSucess ,len(indexlist);

#进行删除操作
def deleteInfo(request):
    #获取要删除的学号
    deletenumber =  request.GET.get('deletenumber');
    print(deletenumber+"@@@@@@@@")
    #将对应的学号的信息中的所带班级进行删除
    deleteResult = User.objects(uid = deletenumber);
    if len(deleteResult) > 0 :  #表示查找到信息
        #更新User中对应的个人信息中的班级内容
        deleteResult = User.objects(uid=deletenumber)[0];
        deleteResult.student_class_name = '';
        #删除对应老师中的助教信息中的内容
        teachUid = request.session.get('uid');
        currentInfo = User.objects(uid = teachUid );
        if len(currentInfo) > 0 :
            currentInfo = User.objects(uid=teachUid)[0];
            dictInfoDel = currentInfo.assisant_info;
            del dictInfoDel[deletenumber];
            currentInfo.save();
            deleteResult.save();  #更新数据
    return HttpResponseRedirect('/assistantmanage/?del='+ 'success'); #重定向回主页面

#进行更新操作
def updataInfo(request):
    #拿到传送过来的数据
    updataNumber = request.POST.get('updataNumber');
    updataName = request.POST.get('updataName');
    updataClass =  request.POST.get('updataClass');

    #更新数据
    updataresult = User.objects(uid = updataNumber);
    if len(updataresult) != 0 :
        updataresult = User.objects(uid=updataNumber)[0]; #更新对应助教的个人信息数据库
        updataresult.uid = updataNumber; #学号
        updataresult.name = updataName; #姓名
        updataresult.student_class_name = updataClass;#所带班级
        #更新对应老师中的助教内容
        teachUid = request.session.get('uid');  #当前老师的uid
        currentInfo = User.objects(uid = teachUid);
        if len(currentInfo) > 0:
            currentInfo = User.objects(uid=teachUid)[0];
            dictInfoDel = currentInfo.assisant_info;
            dictInfoDel[updataNumber] = updataName;
            currentInfo.save();
            updataresult.save();  # 更新数据
    return HttpResponseRedirect('/assistantmanage/?upd='+ 'success');

#ajax请求得到当前老师所带的班级信息，用来动态生成选择框中的内容
def getTeacherBelongClass(request):
    requestResult = ''; #请求的结果
    requestClass = []; #请求得到的班级
    teacherUid = request.session.get('uid'); #得到当前老师的uid信息
    try:
       currentInfo =  User.objects(uid = teacherUid)[0];
       requestClass = currentInfo.teacher_class_name ; #得到所带的班级信息，返回的是列表
       requestResult = 'success';
    except Exception as  errors:
        requestResult = 'fail';
    data = {'requestResult':requestResult , 'classNumberData':requestClass};
    return HttpResponse(JsonResponse(data), content_type="application/json")

#跳转到对应的视图
def jumptoeverymodel(request):
    # 首先判断是否是属于登录中的用户，否则跳转到登录页面中
    isOnlineUser = request.session.get('uid');
    if isOnlineUser == None:  # 表示不是登录中的用户
        return HttpResponseRedirect('/index/');
    else:
        return render(request , 'Csapphnu/userinfomanage.html');

#权限验证各个模块
def userTypeVerification(request):
    if request.method == 'GET':
        #得到要访问的是哪个模块
        modelType = request.GET.get('currentmodeltype');
        usertype = request.session.get('permission');  #从session中得到该用户的权限
        result = '';
        if modelType == 'grounpmodel':      #访问学生分组模块，学生权限不能进
            if int(usertype) <= 1:
                result = 'fail';
            else:
                result = 'success';
        elif modelType == 'assistantmange': #访问老师管理助教模块，学生和助教权限不能进
            if int(usertype) <= 2:
                result = 'fail';
            else:
                result = 'success';
        data = {'result':result};
    return HttpResponse(JsonResponse(data), content_type="application/json")


from pyspark import SparkConf, SparkContext

conf = SparkConf().setMaster("local").setAppName("FriendsByAge")
sc = SparkContext(conf = conf)

def parseLine(line):
    fields = line.split(',')
    age = int(fields[2])
    numFriends = int(fields[3])
    return (age, numFriends)

lines = sc.textFile("fakefriends.csv")
rdd = lines.map(parseLine)

# mapValues - adds an instance marker; could count # of instances a different way, but this is pretty efficient (prevents having to deal with another variable)
#   eg. (33, 385) => (33, (385, 1))
#       (33, 2)   => (33, (2, 1))

# reduceByKey - for each key, combine num_friends 1st element and num_instances 2nd element of all instances
#   updates value, maintains key
#   eg. (33, (385, 1)) and (33, (2, 1)) => (33, (387, 2))
totalsByAge = rdd.mapValues(lambda x: (x, 1)).reduceByKey(lambda x, y: (x[0] + y[0], x[1] + y[1]))
#averagesByAge = totalsByAge.mapValues(lambda x: x[0] / x[1])
averagesByAge = totalsByAge.mapValues(lambda x: (x[0] / x[1], x[1])) # x[1] is # of instances
results = averagesByAge.collect()
for result in results:
    #print(result)
    print("age %d: avg = %d,  %d instances" % (result[0], result[1][0], result[1][1]))


#!/usr/bin/python
# -*- coding: utf-8 -*-
"""
@Time    : 15/11/2018 10:51 PM
@Author  : Jann
@Contact : l33klin@foxmail.com
@Site    :
@File    : test_appium_operate.py
"""
from appium import webdriver

desired_caps = {
  "platformName": "Android",
  "deviceName": "192.168.58.101:5555",
  "app": "/Users/klin/CODE/temp/zmninfo-debug.apk"
}

driver = webdriver.Remote(command_executor="http://127.0.0.1:4723/wd/hub", desired_capabilities=desired_caps)

# 获取Session信息
dp = driver.desired_capabilities

# 获取页面XML结构
page_source = driver.page_source

# 设置超时
driver.set_page_load_timeout(5000)

# 隐式等待
driver.implicitly_wait(5000)

# 设置异步执行脚本超时时间，仅限于Web Context
driver.set_script_timeout(5000)

# 获取手机重力感应的方向，水平或者垂直{LANDSCAPE|PORTRAIT}
orientation = driver.orientation

# 设置手机重力感应方向
driver.orientation = "LANDSCAPE"

#
log_types = driver.log_types

# 获取给定类型的log，每次获取后log缓冲区会被清空
logs = driver.get_log('driver')

# 获取手机设置
settings = driver.get_settings()

# 设置手机
driver.update_settings(settings)

# Activity操作
# 启动Activity
driver.start_activity("com.example", "ActivityName")

# 获取当前Activity
activity = driver.current_activity

# 获取当前Package
package = driver.current_package

# APP操作
# 安装
driver.install_app()

# 是否安装
driver.is_app_installed()

# 启动APP

# 后台运行APP

# 关闭APP

# 重置APP,效果如:adb shell pm clear
driver.reset()

# 卸载APP

# 获取测试覆盖率:https://github.com/appium/appium/blob/master/docs/en/android_coverage.md
driver.end_test_coverage("Intent", "/path")

# 文件操作
# Push文件到手机

# Pull文件到本地

# 交互操作
# 抖动

# 锁屏

# 解锁

# 判断是否锁屏

# 按键操作
# 按下某个或某两个按键:KeyCode:https://developer.android.com/reference/android/view/KeyEvent

# 长按

# 网络操作
# 开关Wifi
driver.toggle_wifi()

# 开关定位服务
driver.toggle_location_services()

# 设置网络
driver.set_network_connection(6)

# 性能数据:Python 不支持
# 获取性能数据

# 录屏
# 开始录屏

# 停止录屏

# 设置粘贴板
driver.set_clipboard('test')

# 获取系统时间
time = driver.device_time()

# Hybrid
# 获取所有的Contexts
contexts = driver.contexts

# 获取当前的Context
context = driver.current_context
# or
context = driver.context

# 切换Context
driver.switch_to.context("NATIVE_APP")

# 移动到控件
from appium.webdriver.common.touch_action import TouchAction
from appium.webdriver.common.multi_action import MultiAction
actions = TouchAction(driver)
actions.press()
actions.move_to()
actions.move_to()
actions.release()
actions.perform()

#
from appium.webdriver.common.touch_action import TouchAction
# ...
driver.scroll()

# 等待
driver.wait_activity()


from django.conf import settings
from django.core.urlresolvers import resolve
from django.http import Http404

import logging

logger = logging.getLogger('jezdez.blog.middleware')

class RenderRstMiddleware(object):
    extension = '.rst'

    def process_response(self, request, response):
        # Only check if there's a 404 for the original response
        if response.status_code != 404 or not request.path.endswith(self.extension):
            return response

        path = request.path[:-len(self.extension)] + '/'
        try:
            view, args, kwargs = resolve(path)
        # Return the original response if any errors happened. Because this
        # is a middleware, we can't assume the errors will be caught elsewhere.
        except Http404:
            return response
        except:
            if settings.DEBUG:
                raise
            return response
        else:
            kwargs['request'] = request
            raw_response = view(*args, **kwargs)

            raw = False
            if 'object' in raw_response.context_data:
                raw_response.template_name = ['zinnia/entry_detail.rst']
                raw = True

            elif 'object_list' in raw_response.context_data:
                raw_response.template_name = ['zinnia/entry_list.rst']
                raw = True

            if raw:
                if settings.DEBUG:
                    content_type = 'text/plain'
                else:
                    content_type = 'text/x-rst'
                raw_response['Content-Type'] = content_type

                if not raw_response.is_rendered:
                    raw_response.render()
                return raw_response
            else:
                return response


import json
import re
import logging
import traceback
from io import BytesIO

import requests
from amazon.api import AmazonAPI
from bs4 import BeautifulSoup
from ebaysdk.trading import Connection as Trading
from PIL import Image

from django.conf import settings
from django.utils import timezone

from .models import AmazonItem

logger = logging.getLogger(__name__)

class Amazon(object):

    def __init__(self):
        try:
            self.connection = AmazonAPI(
                settings.AMAZON_ACCESS_KEY, settings.AMAZON_SECRET_KEY,
                settings.AMAZON_ASSOCIATE_TAG
            )
            logger.info(u'Established Amazon API connection')
        except:
            self.connection = None
            logger.error(traceback.format_exc())
            logger.error(u'Failed to establish Amazon API connection')
        self.total_count = 0

    def get_review_count(self, title, url, has_review, review_url):
        if not has_review:
            return 0
        try:
            response = requests.get(review_url)
            soup = BeautifulSoup(response.content, 'html.parser')
            count = soup.find(string=re.compile('[0-9,]+ customer reviews'))
            count = int(count.split()[0].replace(',', ''))
            logger.info(
                u'Review count for Amazon item {} from url {} is {}'.format(
                    title, url, count
                )
            )
            return count
        except AttributeError:
            logger.warning(
                u'Review count for Amazon item {} from url {} not foun'
                'd'.format(title, url)
                )
            return 0
        except:
            logger.error(traceback.format_exc())
            return 0

    def get_image_list(self, result):
        image_list = []
        for url in [str(image.LargeImage.URL) for image in result.images]:
            response = requests.get(url)
            image = Image.open(BytesIO(response.content))
            height, width = image.size
            if height >= 500 and width >= 500:
                image_list.append(url)
        logger.info(
            u'Got {} images for Amazon item {}'.format(
                len(image_list), result.title
            )
        )
        return json.dumps(image_list)

    def parse_result(self, result, search_obj):
        url = '/'.join(result.offer_url.split('/')[:-1])
        title = result.title
        feature_list = json.dumps(result.features)
        image_list = self.get_image_list(result)
        price = result.price_and_currency[0] or result.list_price[0] or 0
        manufacturer = result.manufacturer
        mpn = result.mpn
        review_count = self.get_review_count(title, url, *result.reviews)
        item = {
            'search': search_obj,
            'url': url,
            'title': title,
            'feature_list': feature_list,
            'image_list': image_list,
            'price': price,
            'manufacturer': manufacturer,
            'mpn': mpn,
            'review_count': review_count,
        }
        message = 'Parsed Amazon item:\n'
        for key, value in item.items():
            message += '{}: {}\n'.format(key.upper(), value)
        logger.info(message.strip())
        return item

    def search(self, search_obj):
        try:
            results = self.connection.search(
                Keywords=search_obj.query, SearchIndex='All'
            )
            results = list(reversed(list(results)))
            logger.info(
                u'Got {} search results from Amazon API for query {}'.format(
                    len(results), search_obj.query
                )
            )
        except:
            results = []
            logger.error(traceback.format_exc())
            logger.warning(
                u'Got 0 search results from Amazon API for query {}'.format(
                    search_obj.query
                )
            )
        search_obj.date_searched = timezone.now()
        search_obj.save()
        count = 0
        for result in results:
            if count > settings.MAX_AMAZON_ITEM_COUNT_PER_SEARCH:
                logger.info(
                    u'Reached maximum Amazon item count per search limit'
                )
                break
            result = self.parse_result(result, search_obj)
            item_obj = AmazonItem(**result)
            if item_obj.is_valid():
                try:
                    item_obj.save()
                    logger.info(
                        u'Saved amazon item: {}'.format(item_obj.title)
                    )
                    count += 1
                except:
                    logger.error(traceback.format_exc())
                    logger.warning(
                        u'Failed to save amazon item: {}'.format(
                            item_obj.title
                        )
                    )
        logger.info(
            u'Saved {} amazon items for query {}'.format(
                count, search_obj.query
            )
        )
        self.total_count += count

class Ebay(object):

    def __init__(self):
        try:
            self.production_connection = Trading(
                devid=settings.EBAY_PRODUCTION_DEVID,
                appid=settings.EBAY_PRODUCTION_APPID,
                certid=settings.EBAY_PRODUCTION_CERTID,
                token=settings.EBAY_PRODUCTION_TOKEN, config_file=None
            )
            logger.info(u'Established Ebay Production API connection')
            self.sandbox_connection = Trading(
                domain='api.sandbox.ebay.com',
                devid=settings.EBAY_SANDBOX_DEVID,
                appid=settings.EBAY_SANDBOX_APPID,
                certid=settings.EBAY_SANDBOX_CERTID,
                token=settings.EBAY_SANDBOX_TOKEN, config_file=None
            )
            logger.info(u'Established Ebay Sandbox API connection')
        except:
            self.production_connection = self.sandbox_connection = None
            logger.error(traceback.format_exc())
            logger.error(u'Failed to establish Ebay API connection')
        self.total_count = 0

    def category_search(self, query):
        response = self.production_connection.execute(
            'GetSuggestedCategories', {'Query': query}
        )
        response = response.dict()
        logger.info(
            u'Got {} suggested categories for query {}'.format(
                response['SuggestedCategoryArray']['SuggestedCategory'],
                query
            )
        )
        return [
            (int(c['Category']['CategoryID']), c['Category']['CategoryName'])
            for c in response['SuggestedCategoryArray']['SuggestedCategory']
        ]

    def list(self, item_obj):
        if settings.USE_SANDBOX:
            connection = self.sandbox_connection
            url = 'http://cgi.sandbox.ebay.com/ws/eBayISAPI.dll?ViewItem&item'\
                '={}&ssPageName=STRK:MESELX:IT'
        else:
            connection = self.produiction_connection
            url = 'http://www.ebay.com/itm/-/{}?ssPageName=ADME:L:LCA:US:1123'
        item_dict = {
            'Item': {
                'Title': item_obj.title,
                'Description': u'<![CDATA[{}]]>'.format(item_obj.html),
                'PrimaryCategory': {'CategoryID': item_obj.category_id},
                'StartPrice': item_obj.price,
                'CategoryMappingAllowed': 'true',
                'ConditionID': '1000',
                'Country': 'US',
                'Currency': 'USD',
                'DispatchTimeMax': '3',
                'ListingDuration': 'Days_30',
                'ListingType': 'FixedPriceItem',
                'Location': 'Los Angeles, CA',
                'PaymentMethods': 'PayPal',
                'PayPalEmailAddress': 'joshwardini@gmail.com',
                'PictureDetails': {
                    'PictureURL': item_obj.get_image_list(),
                },
                'ItemSpecifics': {
                    'NameValueList': [
                        {'Name': 'Brand', 'Value': item_obj.manufacturer},
                        {'Name': 'MPN', 'Value': item_obj.mpn},
                    ]
                },
                'PostalCode': '90001',
                'Quantity': '1',
                'ShippingDetails': {
                    'ShippingType': 'Flat',
                    'ShippingServiceOptions': {
                        'ShippingServicePriority': '1',
                        'ShippingService': 'USPSMedia',
                        'ShippingServiceCost': '0.00'
                    }
                },
                'ReturnPolicy': {
                    'Description': '14 days money back, you pay return shippin'
                    'g',
                    'ReturnsAcceptedOption': 'ReturnsAccepted',
                    'RefundOption': 'MoneyBack',
                    'ReturnsWithinOption': 'Days_14',
                    'ShippingCostPaidByOption': 'Buyer'
                },
                'Site': 'US',
            }
        }
        if item_obj.upc:
            item_dict['Item'] = {
                'ProductListingDetails': {
                    'UPC': item_obj.upc,
                    'ListIfNoProduct': 'true',
                },
            }
        try:
            response = connection.execute('AddFixedPriceItem', item_dict)
            logger.info(u'Listed Ebay item {}'.format(item_obj.title))
            item_obj.url = url.format(response.dict()['ItemID'])
            item_obj.is_listed = True
            item_obj.date_listed = timezone.now()
        except:
            logger.info(u'Failed to list Ebay item {}'.format(item_obj.title))
            item_obj.error = traceback.format_exc()
        try:
            item_obj.save()
            logger.info(u'Saved Ebay item {}'.format(item_obj.title))
            self.total_count += 1
        except:
            logger.info(u'Failed to save Ebay item {}'.format(item_obj.title))


"""
An xml parser, used to get the required data from UserTest.xml and Mapping.xml
"""

__version__ = '1.0'
__author__ = 'kfircohe'
__maintainer__ = ["kfircohe", "amitaysh", "arthurgo"]

import xml.etree.ElementTree as Et

class XmlParser:
    def __init__(self, xml_to_parse):
        """
        Constructor
        :param xml_to_parse: the full path (or name if it is located in the same folder) of the xml you want to parse
        """
        # for UserTestFile.xml
        self.CONFIG = 'Config'
        self.TEST = 'Test'

        # for Mapping.xml
        self.GENERAL_CONFIGURATION = 'GeneralConfiguration'
        self.DELTA_CONFIGURATION = 'DeltaConfiguration'

        self._tree = Et.parse(xml_to_parse)
        self._root = self._tree.getroot()

        self._all_elements = []
        self._append_full_xml_flattened()

    def get_all_elements(self):
        """
        :return: a list of dictionaries containing the entire xml, flattened
        """
        return self._all_elements

    def get_user_configs(self):
        """
        :return: a list of dictionaries containing all the user configs in the file
                 To be used in UserTest.xml
        """
        return self._get_sub_elements(self.CONFIG)

    def get_user_tests(self):
        """
        :return: a list of dictionaries containing all the tests in the file
                 To be used in UserTest.xml
        """
        return self._get_sub_elements(self.TEST)

    def get_mapping_general_configuration(self, architecture):
        """
        :param architecture: the CPU architecture (e.g., CNL, ICL, GLK, etc..)
        :return: a list of dictionaries containing all the general configurations of a specific architecture
                 To be used in Mapping.xml
        """
        return self._get_architecture_configuration(architecture, self.GENERAL_CONFIGURATION)

    def get_mapping_delta_configuration(self, architecture):
        """
        :param architecture: the CPU architecture (e.g., CNL, ICL, GLK, etc...)
        :return: a list of dictionaries containing all the delta configurations of a specific architecture
                 To be used in Mapping.xml
        """
        return self._get_architecture_configuration(architecture, self.DELTA_CONFIGURATION)

    def _get_sub_elements(self, element_to_find):
        """
        Gets the first level of sub elements of a certain element

        :param element_to_find: the element you want to retrieve its sub elements
        :return: a list of dictionaries containing the sub elements of the selected element
        """
        for sub_element in self._all_elements:
            if sub_element.get(element_to_find) is not None:
                return sub_element.get(element_to_find)

    def _get_architecture_configuration(self, architecture, configuration):
        """
        :param architecture: the CPU architecture (e.g., CNL, ICL, GLK, etc..)
        :param configuration: the CPU configuration (e.g., U8, A2, U6, etc..)
        :return: a list of dictionaries containing all the configurations of a specific architecture and configuration
                 (e.g., 'CNL U8', 'ICL A2', etc...)
        """
        for config in self._root:
            if config.find(architecture) is not None:
                for sub_element in config.find(architecture):
                    if sub_element.tag == configuration:
                        return self._append_all_sub_elements(sub_element)

    def _append_full_xml_flattened(self):
        """
        :return: a list of dictionaries containing the entire xml, flattened
        """
        for node in self._root:
            temp_sub_elements = []
            for sub_element in node:
                if self._is_element_empty(sub_element):
                    temp_sub_elements.append({sub_element.tag: self._append_all_sub_elements(sub_element)})
                else:
                    temp_sub_elements.append({sub_element.tag: sub_element.text})

            self._all_elements.append({node.tag: temp_sub_elements})

    def _append_all_sub_elements(self, element_to_find):
        """
        Recursively gets all sub elements of a certain element, until there are no more sub elements

        :param element_to_find: the element you want to retrieve its sub elements
        :return: a list of dictionaries containing all the sub elements of a certain element
        """
        all_subs = []
        for sub in list(element_to_find):
            if self._is_element_empty(sub):
                all_subs.append({sub.tag: self._append_all_sub_elements(sub)})
            else:
                all_subs.append({sub.tag: sub.text})
        return all_subs

    @staticmethod
    def _is_element_empty(element_to_find):
        """
        :param element_to_find: the element you want to retrieve its sub elements
        :return: True, if element is empty (i.e., it has no value)
        """
        if element_to_find.text.strip('\n\t ') == '':
            return True

# if __name__ == '__main__':
#     xps = XmlParser('Mapping.xml')
#     a = xps.get_mapping_delta_configuration('CNL')
#     b = xps.get_mapping_general_configuration('ICL')
#
#     xps = XmlParser('ExampleUserTest.xml')
#     a = xps.get_user_tests()
#     b = xps.get_user_configs()
#
#     print "C:"
#     for element in a:
#         print element
#
#     print "D:"
#     for element in b:
#         print element


#Hands on #2 | #1
print("-----Hands On #2 | #6-----\n")

import random
prompt = "\nWould you like to continue?"
prompt += " Enter 'q' to end the program or press enter to continue.\n"

keepGoing = ""
counter=0
numberCorrect=0

while keepGoing != "q":
    randNum1 = random.randrange(1,1000)
    randNum2 = random.randrange(1,1000)
    answer = int(randNum1 + randNum2)
    questionNumber = counter + 1

    inputAnswer = int(input(f"{questionNumber}) What is the answer to {randNum1} + {randNum2}?"))
    if answer==inputAnswer:
        print("   > Correct!")
        numberCorrect+=1
    else:
        print(f"   > Your answer is incorrect. Correct answer: {answer}.")
    counter+=1
    keepGoing = input(prompt)

print(f"\nQuestions answered correctly: {numberCorrect}.")


# matplotlib-based plotting script for Iliev et al. test #2
# Daniel R. Reynolds, reynolds@smu.edu

# imports
from pylab import *
from yt.mods import *
from os import *

# reduce log level of yt
from yt.config import ytcfg
ytcfg["yt","loglevel"] = "50"

# set the total number of snapshots
te = 50

# set the graphics output type
#pictype = 'pdf'
pictype = 'png'

# set some constants
Ngammadot = 5.0e48     # ionization source strength [photons/sec]
aHII = 2.52e-13        # recombination rate coefficient
mp = 1.67262171e-24    # proton mass [g]
Myr = 3.15576e13       # duration of a Megayear [sec]
nH = 1.0e-3            # input hydrogen number density [cm^(-3)]
trec = 1.0/(aHII*nH)   # recombination time [sec]
rs0 = (3.0*Ngammadot/4/pi/aHII/nH/nH)**(1.0/3.0)   # Stromgren radius

# Define some derived fields
#   Neutral Hydrogen fraction (log plot)
def _xHI(field, data):
    return (data["HI_Density"]/data["Density"])
add_field("xHI", take_log=True, function=_xHI,
          display_name="Neutral\; Fraction")

#   Ionized Hydrogen fraction (log plot)
def _xHII(field, data):
    return (data["HII_Density"]/data["Density"])
add_field("xHII", take_log=True, function=_xHII,
          display_name="Ionized\; Fraction")

#   Radiation energy density (log plot)
def _logE(field, data):
    return (data["Grey_Radiation_Energy"])
def _convertlogE(data):
    return ( data.convert("MassUnits")/data.convert("TimeUnits") /
             data.convert("TimeUnits")/data.convert("cm"))
add_field("logE", take_log=True, function=_logE,
          convert_function=_convertlogE,
          display_name="Radiation\; Energy\; Density",
          units=r"\rm{erg}/\rm{cm}^3")

#   Radius from domain center
def _radius(field, data):
    return (np.sqrt(data["x"]*data["x"] + data["y"]*data["y"] +
                    data["z"]*data["z"]))
add_field("radius", take_log=False, function=_radius,
          display_name="radius", units=r"{r/L_{box}}")

# initialize time-history outputs
#    row 1: time (t)
#    row 2: computed i-front radius
#    row 3: predicted i-front radius (rI)
#    row 4: stromgren sphere radius (rs)
rdata = zeros( (4, te+1), dtype=float);

# loop over snapshots, loading values and times
for tstep in range(0,te+1):

    # load relevant information
    sdump = repr(tstep).zfill(4)
    pfile = 'DD' + sdump + '/data' + sdump
    pf = load(pfile)
    t = pf.current_time * pf["TimeUnits"]

    # determine if simulation was run with source in center or corner
    spherical = (2.0**(pf.domain_left_edge[0]/pf.domain_right_edge[0]+1.0)
               * 2.0**(pf.domain_left_edge[1]/pf.domain_right_edge[1]+1.0)
               * 2.0**(pf.domain_left_edge[2]/pf.domain_right_edge[2]+1.0))

    # compute I-front radius (assuming spherical)
    sp = pf.h.sphere([0.0, 0.0, 0.0], 1.0)
    HIIvolume = (sp["xHII"]*sp["CellVolumeCode"]*pf["cm"]**3).sum()*spherical
    radius = (3.0/4.0*HIIvolume/pi)**(1.0/3.0)

    # compute analytical solution
    ranal = rs0*(1.0 - exp(-t/trec))**(1.0/3.0)

    # store data
    rdata[0][tstep] = t/trec
    rdata[1][tstep] = radius
    rdata[2][tstep] = ranal
    rdata[3][tstep] = rs0

    # generate 2D plots at certain times
    if (tstep == 1) or (tstep == 5) or (tstep == 10) or (tstep == 20) or (tstep == 50):

        # set time label
        if (tstep == 1):
            Myr = '010'
        elif (tstep == 5):
            Myr = '050'
        elif (tstep == 10):
            Myr = '100'
        elif (tstep == 20):
            Myr = '200'
        else:
            Myr = '500'

        # determine domain "center" for plotting
        xC = 0.5*(pf["DomainLeftEdge"][0] + pf["DomainRightEdge"][0])
        yC = 0.5*(pf["DomainLeftEdge"][1] + pf["DomainRightEdge"][1])
        zC = 0.5*(pf["DomainLeftEdge"][2] + pf["DomainRightEdge"][2])

        # begin plot collection
        pc = PlotCollection(pf, [xC,yC,0.0])

        # xHI slice through z=0
        p = pc.add_slice("xHI",'z')
        p.modify["title"]('HI fraction, t =' + Myr + ' Myr')

        # xHII slice through z=0
        p = pc.add_slice("xHII",'z')
        p.modify["title"]('HII fraction, t =' + Myr + ' Myr')

        # E slice through z=0
        p = pc.add_slice("logE",'z')
        p.modify["title"]('radiation density, t =' + Myr + ' Myr')

        pc.save(Myr + 'Myr', format=pictype)

        # rename generated files
        f1 = Myr + 'Myr_Slice_z_xHI.' + pictype
        f2 = 'HIcontour_' + Myr + 'Myr.' + pictype
        rename(f1,f2)
        f1 = Myr + 'Myr_Slice_z_xHII.' + pictype
        f2 = 'HIIcontour_' + Myr + 'Myr.' + pictype
        rename(f1,f2)
        f1 = Myr + 'Myr_Slice_z_logE.' + pictype
        f2 = 'Econtour_' + Myr + 'Myr.' + pictype
        rename(f1,f2)

        # generate profile plots by averaging results from multiple rays
#         rays = np.array( ((1.0,0.0,0.0), (0.0,1.0,0.0), (0.0,0.0,1.0),
#                           (-1.0,0.0,0.0), (0.0,-1.0,0.0), (0.0,0.0,-1.0),
#                           (1.0,1.0,0.0), (1.0,0.0,1.0), (0.0,1.0,1.0),
#                           (-1.0,1.0,0.0), (-1.0,0.0,1.0), (0.0,-1.0,1.0),
#                           (1.0,-1.0,0.0), (1.0,0.0,-1.0), (0.0,1.0,-1.0),
#                           (-1.0,-1.0,0.0), (-1.0,0.0,-1.0), (0.0,-1.0,-1.0),
#                           (1.0,1.0,1.0), (-1.0,1.0,1.0), (1.0,-1.0,1.0),
#                           (1.0,1.0,-1.0), (-1.0,-1.0,1.0), (-1.0,1.0,-1.0),
#                           (1.0,-1.0,-1.0), (-1.0,-1.0,-1.0)) )
#         nrays = 26
        rays = np.array( ((1.0,0.0,0.0), (0.0,1.0,0.0), (0.0,0.0,1.0),
                          (1.0,1.0,0.0), (1.0,0.0,1.0), (0.0,1.0,1.0),
                          (1.0,1.0,1.0)) )
        nrays = 7
        nradii = 200
        rvals = np.linspace(0,1*pf["cm"]/rs0,nradii)
        HIprofile  = np.zeros(rvals.shape, dtype=float)
        HIIprofile = np.zeros(rvals.shape, dtype=float)

        # generate 1D profiles from ray emanating out from box center to corner
        for iray in range(0,nrays):
            rvec = rays[iray,:]
            rvec = rvec / sqrt(rvec[0]**2 + rvec[1]**2 + rvec[2]**2)
            r = pf.h.ray([0.0, 0.0, 0.0], rvec)
            HIprof  = log10(r["xHI"])
            HIIprof = log10(r["xHII"])
            Hradii  = r["radius"]

            # sort results by radius (since that isn't quite working correctly from yt)
            ptype = [('r', float), ('xHI', float), ('xHII', float)]
            pdata = np.zeros(Hradii.shape, dtype=ptype);
            nrad = (Hradii.shape)[0]
            for irad in range(0,nrad):
                pdata[irad] = (Hradii[irad], HIprof[irad], HIIprof[irad])
            pdata = np.sort(pdata, order='r')

            # interpolate results into output arrays
            tmp = np.interp(rvals, pdata['r'], pdata['xHI'])
            HIprofile += tmp
            tmp = np.interp(rvals, pdata['r'], pdata['xHII'])
            HIIprofile += tmp
        HIprofile  /= nrays
        HIIprofile /= nrays

        # chemistry profiles
        figure()
        plot(rvals,HIprofile,'b-',rvals,HIIprofile,'r--')
        grid()
        xlabel('$r/L_{box}$')
        ylabel('log(xHI), log(xHII)')
        title('HI, HII Profiles, t =' + Myr + ' Myr')
        legend( ('xHI','xHII'), 'lower right' )
        axis([ 0.0, 1.2, -7.0, 1.0 ])
        savefig('profiles_' + Myr + 'Myr.' + pictype)

# I-front radius/velocity plots
figure()
plot(rdata[0],rdata[1]/rs0,'b-',rdata[0],rdata[2]/rs0,'r--')
xlabel('$t/t_{rec}$')
ylabel('$r_I/r_S$')
title('Propagation of HII Region')
legend( ('computed', 'analytical'), loc=4 )
grid()
axis([ 0.0, 4.5, 0.0, 1.1 ])
savefig('rad_vs_time.' + pictype)

figure()
times = (rdata[0][1:te] + rdata[0][2:te+1])/2.0
velocity = (rdata[1][2:te+1] - rdata[1][1:te])/(rdata[0][2:te+1]-rdata[0][1:te])/rs0
vel_anal = (rdata[2][2:te+1] - rdata[2][1:te])/(rdata[0][2:te+1]-rdata[0][1:te])/rs0
plot(times,velocity,'b-',times,vel_anal,'r--')
xlabel('$t/t_{rec}$')
ylabel('$v_I/(r_S/t_{rec})$')
title('Velocity of HII Region')
legend( ('computed', 'analytical') )
grid()
axis([ 0.0, 4.5, -0.2, 1.2 ])
savefig('vel_vs_time.' + pictype)