# Python 3.7

''' data.csv
1,30000,20200101,20200130
2,30000,20200102,20200131
3,29000,20200103,20200201
'''

import sys
import datetime
import traceback

DATE_FMT = '%Y%m%d'

def load_data():
    '''将数据改为字段并处理时间'''

    datas = []
    header = ['user_id', 'cost', 'start_day', 'end_day']

    # 从文件加载数据
    with open('data.csv', 'r') as f:
        datas = f.readlines()
    datas = [i.strip().split(',') for i in datas]

    # 处理数据
    ret = []
    for i in datas:

        # 将数据处理为字典
        data = {}
        for index, value in enumerate(i):
            data[header[index]] = i[index]

        # 简单的转换一下格式
        try:
            data['start_day'] = datetime.datetime.strptime(data['start_day'], DATE_FMT)
            data['end_day'] = datetime.datetime.strptime(data['end_day'], DATE_FMT)
            data['user_id'] = int(data['user_id'])
            data['cost'] = int(data['cost'])
        except:
            print(f'error data {data}\n {traceback.format_exc()}')
            continue
        ret.append(data)
    return ret

def get_one_cost(data, calc_date):
    '''假设用户购买会员价格存在不同'''

    cost = 0

    # 过期或者未到期不可收取收益
    if calc_date < data['start_day'] or calc_date > data['end_day']:
        return cost

    # 根据购买时长和金额计算单价收益
    days = (data['end_day'] - data['start_day']).days + 1
    income = data['cost'] // days
    rest = data['cost'] % days

    # 根据计算时间判断收益
    cost = income
    if calc_date == data['end_day']:
        cost += rest

    return cost

def main():

    # 校验输入
    if len(sys.argv) != 2:
        print('use as `python3.7 cost.py 20200102`')
        return

    # 转换计算时间格式
    calc_date = sys.argv[1]

    try:
        calc_date = datetime.datetime.strptime(calc_date, DATE_FMT)
    except ValueError:
        print('not correct input date, use like 20200102(%Y%m%d)')
    except:
        print(f'other error {traceback.format_exc()}')

    # 计算
    datas = load_data()
    total = 0

    for i in datas:
        cost = get_one_cost(i, calc_date)
        print(f'use data {i} calc: user {i["user_id"]} {cost}')
        total += cost
    print(f'total {total}')

if __name__ == '__main__':
    main()


import numpy as np
import cv2
import cv2.aruco as aruco
import glob

criteria = (cv2.TERM_CRITERIA_EPS + cv2.TERM_CRITERIA_MAX_ITER, 30, 0.001)
#cap = cv2.VideoCapture(cv2.CAP_DSHOW)
cap = cv2.VideoCapture(10)

# set opencv videocapture properties for width, height and fps
cap.set(cv2.CAP_PROP_AUTOFOCUS, 0)
cap.set(cv2.CAP_PROP_FRAME_WIDTH, 3840)
cap.set(cv2.CAP_PROP_FRAME_HEIGHT, 2160)
cap.set(cv2.CAP_PROP_FPS, 30)

while (True):
    ret, frame = cap.read()

    # display the resulting frame
    cv2.imshow('frame', frame)
    if cv2.waitKey(1) & 0xFF == ord('q'):
        break

# When everything done, release the capture
cap.release()
cv2.destroyAllWindows()


import configparser
indicator = [1,3,4,6,7]
def create_config(indicator):
    config = configparser.ConfigParser()
    config.read('./NEAT-Try/XOR-conf')
    print("Loading config!")
    config['DefaultGenome']['num_inputs'] = str(len(indicator))
    config.add_section('Inputs')
    config.set('Inputs','Indicators', str(indicator))
    with open('temp.conf','w') as f:
        config.write(f)
    print("Temp config created!")

create_config(indicator)


"""Tracker - physical particle tracker"""

# ------------------------------------
# tracker.py
# Part of the LADiM Model
#
# Bjørn Ådlandsvik, <bjorn@imr.no>
# Institute of Marine Research
#
# Licensed under the MIT license
# ------------------------------------
from __future__ import annotations

import logging
from typing import TYPE_CHECKING, Any

import numba  # type: ignore
import numpy as np

if TYPE_CHECKING:
    from ladim.forcing import BaseForce

ParticleArray = np.ndarray  # 1D array, one element per particle
Velocity = tuple[ParticleArray, ParticleArray]

PARALLEL = False
DEBUG = False

logger = logging.getLogger(__name__)
if DEBUG:
    logger.setLevel(logging.DEBUG)

class Tracker:
    """The physical particle tracking kernel"""

    def __init__(
        self,
        advection: str,  # EF, RK2, or RK4
        diffusion: float = 0.0,
        vertdiff: float = 0.0,
        vertical_advection: bool = False,
        *,
        modules: dict[str, Any]
        # modules: Optional[dict[str, Any]] = None,
    ) -> None:
        logger.info("Initiating the particle tracker")
        self.modules = modules
        self.dt = modules["time"].dt / np.timedelta64(1, "s")
        self.advection = advection  # Name of advection method
        logger.info("  Advection method: %s", advection)

        # advect <- requested advection method
        # advection = string "EF", "RK2", "RK4"
        # advect = the actual method
        if self.advection:
            self.advect = getattr(self, self.advection)

        self.vertical_advection = vertical_advection
        if vertical_advection:
            logger.info("  Vertical advection activated")

        self.diffusion = bool(diffusion)
        self.D = diffusion
        if self.diffusion:
            logger.info("  Horizontal diffusion: %s m²/s", diffusion)

        self.vertdiff = bool(vertdiff)
        self.Dz = vertdiff
        if self.vertdiff:
            logger.info("  Vertical diffusion: %s m²/s", vertdiff)

        self.rng = np.random.default_rng()

    def update(self) -> None:
        """Move the particles one time step"""

        state = self.modules["state"]
        grid = self.modules["grid"]
        force = self.modules["forcing"]

        X, Y, Z = state.X, state.Y, state.Z

        self.dx, self.dy = grid.metric(X, Y)

        self.xmin = grid.xmin + 0.01
        self.xmax = grid.xmax - 0.01
        self.ymin = grid.ymin + 0.01
        self.ymax = grid.ymax - 0.01

        U = np.zeros_like(X)
        V = np.zeros_like(Y)

        # --- Advection ---
        if self.advection:
            Uadv, Vadv = self.advect(X, Y, Z, force)
            U += Uadv
            V += Vadv

        # --- Diffusion ---
        if self.diffusion:
            Udiff, Vdiff = self.diffuse(num_particles=len(X))
            U += Udiff
            V += Vdiff

        # --- Move the particles

        # New position, if OK
        X1 = X + U * self.dt / self.dx
        Y1 = Y + V * self.dt / self.dy

        # Kill particles trying to move out of the grid
        out_of_grid = ~grid.ingrid(X1, Y1)
        state.alive[out_of_grid] = False
        state.active[out_of_grid] = False  # Not necessary if they are removed

        # Do not move inactive particles
        inactive = ~state.active
        X1[inactive] = X[inactive]
        Y1[inactive] = Y[inactive]

        # Land, boundary treatment. Do not move the particles onto land
        # Consider a sequence of different actions
        onland = ~grid.atsea(X1, Y1)
        X1[onland] = X[onland]
        Y1[onland] = Y[onland]

        # Update the particle positions
        state["X"] = X1
        state["Y"] = Y1

        # --- Vertical movement ---

        # Sample the depth level
        h = None
        if self.vertdiff or self.vertical_advection:
            # if hasattr(grid, "sample_depth") and callable(grid.sample_depth):
            #     h = grid.sample_depth(X, Y)
            # elif hasattr(grid, "depth") and callable(grid.depth):
            #     h = grid.depth(X, Y)
            h = grid.depth(X, Y)

            # Diffusion
            if self.vertdiff:
                W = self.diffuse_vert(num_particles=len(X))
                Z += W * self.dt

            # Advection
            if self.vertical_advection:
                W = force.variables["w"]
                Z += W * self.dt

            # Reflexive boundary conditions at surface
            Z[Z < 0] *= -1

            # Reflexive boundary conditions at bottom
            if h is not None:
                below_seabed = Z > h
                Z[below_seabed] = 2 * h[below_seabed] - Z[below_seabed]

            # Update particle positions
            state["Z"] = Z

    def EF(
        self, X: ParticleArray, Y: ParticleArray, Z: ParticleArray, force: BaseForce
    ) -> Velocity:
        """Euler-Forward advective velocity"""

        U, V = force.velocity(X, Y, Z)

        return U, V

    def RK2(
        self, X: ParticleArray, Y: ParticleArray, Z: ParticleArray, force: BaseForce
    ) -> Velocity:
        """Runge-Kutta second order = Heun scheme

        This version does not sample velocities outside the grid
        """

        dt = self.dt
        dtdx = dt / self.dx
        dtdy = dt / self.dy

        U, V = force.velocity(X, Y, Z)
        X1, Y1 = RKstep(X, Y, U, V, 0.5, dtdx, dtdy)
        clip(X, Y, self.xmin, self.xmax, self.ymin, self.ymax)

        return force.velocity(X1, Y1, Z, fractional_step=0.5)

    def RK4(
        self, X: ParticleArray, Y: ParticleArray, Z: ParticleArray, force: BaseForce
    ) -> Velocity:
        """Runge-Kutta fourth order advection

        This version does not sample velocities outside the grid

        """

        dt = self.dt
        dtdx = dt / self.dx
        dtdy = dt / self.dy
        xmin, xmax, ymin, ymax = self.xmin, self.xmax, self.ymin, self.ymax

        U1, V1 = force.velocity(X, Y, Z, fractional_step=0.0)
        X1, Y1 = RKstep(X, Y, U1, V1, 0.5, dtdx, dtdy)
        clip(X1, Y1, xmin, xmax, ymin, ymax)

        U2, V2 = force.velocity(X1, Y1, Z, fractional_step=0.5)
        X2, Y2 = RKstep(X, Y, U2, V2, 0.5, dtdx, dtdy)
        clip(X2, Y2, xmin, xmax, ymin, ymax)

        U3, V3 = force.velocity(X2, Y2, Z, fractional_step=0.5)
        X3, Y3 = RKstep(X, Y, U3, V3, 1.0, dtdx, dtdy)
        clip(X3, Y3, xmin, xmax, ymin, ymax)

        U4, V4 = force.velocity(X3, Y3, Z, fractional_step=1.0)

        return RK4avg(U1, U2, U3, U4), RK4avg(V1, V2, V3, V4)

    def diffuse(self, num_particles: int) -> Velocity:
        """Random walk diffusion"""

        # Diffusive velocity
        stddev = (2 * self.D / self.dt) ** 0.5
        U = stddev * self.rng.normal(size=num_particles)
        V = stddev * self.rng.normal(size=num_particles)

        return U, V

    def diffuse_vert(self, num_particles: int) -> ParticleArray:
        """Random walk diffusion"""

        # Diffusive velocity
        stddev = (2 * self.Dz / self.dt) ** 0.5
        W: ParticleArray = stddev * self.rng.normal(size=num_particles)
        return W

@numba.njit(parallel=PARALLEL)
def RKstep0(
    X: ParticleArray,
    Y: ParticleArray,
    U: ParticleArray,
    V: ParticleArray,
    frac: float,
    dtdx: ParticleArray,
    dtdy: ParticleArray,
) -> tuple[ParticleArray, ParticleArray]:
    Xp = X + frac * U * dtdx
    Yp = Y + frac * V * dtdy
    return Xp, Yp

@numba.njit(parallel=False)
def RKstep1(
    X: ParticleArray,
    Y: ParticleArray,
    U: ParticleArray,
    V: ParticleArray,
    frac: float,
    dtdx: ParticleArray,
    dtdy: ParticleArray,
) -> Velocity:
    """Do a forward Runge-Kutta (partial) step"""

    N = len(X)
    Xp = np.empty(N)
    Yp = np.empty(N)
    for i in numba.prange(N):
        Xp[i] = X[i] + frac * U[i] * dtdx[i]
        Yp[i] = Y[i] + frac * V[i] * dtdy[i]
    return X, Y

RKstep = RKstep1

@numba.njit(parallel=PARALLEL)
def clip(
    X: ParticleArray,
    Y: ParticleArray,
    xmin: float,
    xmax: float,
    ymin: float,
    ymax: float,
) -> None:
    """Clip particle positions in place towards forcing domain boundary"""
    for p in numba.prange(len(X)):
        X[p] = max(min(X[p], xmax), xmin)
        Y[p] = max(min(Y[p], ymax), ymin)

@numba.njit(parallel=PARALLEL)
def RK4avg(
    U1: ParticleArray, U2: ParticleArray, U3: ParticleArray, U4: ParticleArray
) -> ParticleArray:
    """Average velocity component for Runge-Kutta 4-th order"""
    return (U1 + 2 * U2 + 2 * U3 + U4) / 6.0


from babel.messages import frontend as babel
from Cython.Build import cythonize
from Cython.Distutils import Extension
from setuptools import setup

extra_compile_args = ["-O3", "-ffast-math", "-fopenmp"]
extra_link_args = ["-fopenmp"]

lib_modules = []

lib_modules.append(
    Extension(
        "py_extract.zip_decrypter",
        ["py_extract/zip_decrypter.pyx"],
        language="c",
        extra_compile_args=extra_compile_args,
        extra_link_args=extra_link_args,
    ),
)

setup(
    name="zip_decrypter",
    zip_safe=False,
    ext_modules=cythonize(
        lib_modules,
        language_level=3,
        compiler_directives={"always_allow_keywords": True},
    ),
    cmdclass={"compile_catalog": babel.compile_catalog},
)


# 라이브러리 읽어들이기
from bs4 import BeautifulSoup

# 분석하고 싶은 HTML
html = """

스크래핑이란

웹페이지를 분석하는것

원하는 부분을 추출하는것

"""

# HTml 분석하기
soup = BeautifulSoup(html, 'html.parser')

# 원하는 부분 추출하기
h1 = soup.html.body.h1
p1 = soup.html.p
p2 = p1.next_sibling.next_sibling

h1_find = soup.find(id="title")
p_find = soup.find(id="body")

# 요소 글자 출력하기
print("h1 = " + h1.string)
print("p1 = " + p1.string)
print("p2 = " + p2.string)

print("h1_find = " + h1_find.string)
print("p_find = " + p_find.string)


#!/usr/bin/python3.6
from locust import HttpUser, task, between

class QuickstartUser(HttpUser):
    @task
    def get_persons(self):
        endpoint = "/api/person/all"
        headers = {
            "User-Agent": "BASF-LT",
            "Accept": "*/*",
            "Authorization": "Basic dXNlcjpwd2Qh"
        }
        response = self.client.get(endpoint, headers=headers)
        if response.status_code != 200:
            result = False
        # print(response.json()) - To verify that API is returning the list of persons

    wait_time = between(1, 2)


import requests

BASE = "http://127.0.0.1:5000/"
data = [{"likes": 20, "name": 'video 2', "views": 2000}, {"likes": 30, "name": 'video 3', "views": 3000}, {"likes": 40, "name": 'video 4', "views": 4000}]
for i in range(len(data)):
    response = requests.put(BASE + f"video/" + str(i), data[i])
    print(response.json())

input('enter')
response = requests.get(BASE + "video/10")
print(response)


## set warna dan marker

import numpy as np
import matplotlib.pyplot as plt

"""
   1.membuat data
   2. membuat plot
   3. menampilkan plot
"""

# 1. membuat plot (sin(2wt + theta))
# camel case
def sinusGenerator(amplitudo,frekuensi,tAhkhir,theta):
    t = np.arange(0,tAhkhir,0.1)
    y = amplitudo * np.sin(2*frekuensi*t + np.deg2rad(theta))
    return t,y

# 2. membuat plot
t1,y1 = sinusGenerator(1,1,4,0)
plt.plot(t1,y1)

t2,y2 = sinusGenerator(1,1,4,30)
plt.plot(t2,y2, 'r') #red

t3,y3 = sinusGenerator(1,1,4,60)
plt.plot(t3,y3, 'b--') # blue --

t4,y4 = sinusGenerator(1,1,4,90)
plt.plot(t4,y4, 'g-*') #green -o

# 3. menampilkan plot
plt.show()

02 warna dan marker/2warna_and_marker.py


# -*- coding: utf-8 -*-
# @Time : 2022/7/19 16:03
# @Author : chensi
# @File : apiMethod.py
# @Project : CVATest

""" 接口基础方法 """

import os
import json
import logging
import random
import requests
import simplejson
from requests_toolbelt import MultipartEncoder
from config import API_CONFIG, PROJECT_NAME
from common.utils.readYaml import write_yaml_data, read_yaml_data

def get(headers, address, data, timeout=8, cookies=None):
    """
    get请求
    :param headers:
    :param address:
    :param data:
    :param timeout:
    :param cookies:
    :return:
    """

    response = requests.get(headers=headers, url=address, params=data, timeout=timeout, cookies=cookies, verify=False)
    if response.status_code == 301:
        response = requests.get(url=response.headers["location"], verify=False)

    try:
        return response.status_code, response.json()
    except json.decoder.JSONDecodeError:
        return response.status_code, None
    except simplejson.error.JSONDecodeError:
        return response.status_code, None
    except Exception as e:
        logging.exception('ERROR')
        logging.error(e)
        raise

def post(headers, address, mime_type, timeout=8, data=None, files=None, params=None, cookies=None):
    """
    post请求,具体的请求体内部进行处理分类
    :param headers:
    :param address:
    :param mime_type:   请求参数格式  根据此参数对请求体进行区分
    :param timeout:
    :param data:
    :param files:   form-data请求方式需要使用files参数
    :param params:
    :param cookies:
    :return:
    """

    # 判断请求参数类型
    if 'form_data' in mime_type:
        for key in files:
            value = files[key]
            # 判断参数值是否为文件,如果是文件则替换为二进制
            if '/' in value:
                files[key] = (os.path.basename(value)), open((value, 'rb'))
        enc = MultipartEncoder(
            fields=files,
            boundary='-------------------' + str(random.randint(1e28, 1e29 - 1))
        )

        headers['Content-Type'] = enc.content.type
        response = requests.post(url=address, data=enc, headers=headers, timeout=timeout, coookies=cookies,
                                 params=params, verify=False)
    elif 'application/json' in mime_type:
        response = requests.post(url=address, headers=headers, timeout=timeout, cookies=cookies, files=files,
                                 params=params, verify=False)
    else:
        response = requests.post(url=address, json=data, headers=headers, timeout=timeout, files=files, params=params,
                                 cookies=cookies, verify=False)

    try:
        if response.status_code != 200:
            return response.status_code, response.text
        else:
            return response.status_code, response.json()
    except json.decoder.JSONDecodeError:
        return response.status_code, None
    except simplejson.errors.JSONDecodeError:
        return response.status_code, None
    except Exception as e:
        logging.exception('ERROR')
        logging.error(e)
        raise

def put(headers, address, mime_type, timeout, data=None, files=None, params=None, cookies=None):
    """

    :param headers:
    :param address:
    :param mime_type:  请求参数格式（form_data,raw）
    :param timeout:
    :param data:
    :param files:
    :param params:
    :param cookies:
    :return:
    """
    if mime_type == 'raw':
        data = json.dumps(data)
    elif mime_type == 'application/json':
        data = json.dumps(data)
    response = requests.put(url=address,
                            data=data,
                            headers=headers,
                            timeout=timeout,
                            files=files,
                            cookies=cookies,
                            verify=False)
    try:
        return response.status_code, response.json()
    except json.decoder.JSONDecodeError:
        return response.status_code, None
    except simplejson.errors.JSONDecodeError:
        return response.status_code, None
    except Exception as e:
        logging.exception('ERROR')
        logging.error(e)
        raise

def delete(headers, address, data, timeout=8, cookies=None):
    """
    delete请求
    :param headers: 请求头
    :param address: 请求地址
    :param data: 请求参数
    :param timeout: 超时时间
    :param cookies:
    :return:
    """
    response = requests.delete(url=address,
                               params=data,
                               headers=headers,
                               timeout=timeout,
                               cookies=cookies,
                               verify=False)
    try:
        return response.status_code, response.json()
    except json.decoder.JSONDecodeError:
        return response.status_code, None
    except simplejson.errors.JSONDecodeError:
        return response.status_code, None
    except Exception as e:
        logging.exception('ERROR')
        logging.error(e)
        raise

def save_cookie(headers, address, mine_type, time_out=8, data=None, files=None, cookies=None):
    """
    保存cookie信息
    :param headers: 请求头
    :param address: 请求地址
    :param mine_type: 请求类型,用于区分参数格式(form_data,raw)
    :param time_out: 超时时间
    :param data:  请求参数
    :param files: 文件路径
    :param cookies:
    :return:
    """

    if 'data' in mine_type:
        response = requests.post(url=address, data=data, headers=headers, timeout=time_out, files=files, cookies=cookies, verify=False)
    else:
        response = requests.post(url=address, json=data, headers=headers, timeout=time_out, files=files, cookies=cookies, verify=False)

    try:
        cookies = response.cookies.get_dict()
        # 读取api配置并且写入最新的cookie信息
        aconfig = read_yaml_data(API_CONFIG)
        aconfig[PROJECT_NAME]['cookies'] = cookies
        write_yaml_data(API_CONFIG, aconfig)
        logging.debug("cookies已保存,结果为: {}".format(cookies))
    except json.decoder.JSONDecodeError:
        return response.status_code, None
    except simplejson.errors.JSONDecodeError:
        return response.status_code, None
    except Exception as e:
        logging.exception('ERROR')
        logging.error(e)
        raise


import matplotlib.pyplot as plt

def addsig(
    x1, x2, y, t='', h=1.0, lw=1.0, lc='black',
    tc='black', ax=None, figsize=None, fontsize=None
):
    """
    Add signifiance annotation between two groups in a box plot.

    Parameters
    ----------
    x1 : float
        Position of the first box.
    x2 : float
        Position of the second box.
    y : float
        Bottom position of the drawing.
    t : str, default: ''
        Text.
    h : float, default: 1.0
        Height of the drawing.
    lw : float, default: 1.0
        Line width.
    lc : str, default: 'black'
        Line color.
    tc : str, default: 'black'
        Text color.
    ax : matplotlib.axes.Axes, optional
        Axes object to draw the plot onto, otherwise uses the current Axes.
    figsize : tuple, optional
        Width, height in inches. Format: (float, float).
    fontsize : float, optional
        Sets the fontsize.

    Returns
    -------
    matplotlib.axes.Axes
        Axes object with the plot drawn onto it.

    Examples
    --------

    .. code:: python3

        import dokdo
        import matplotlib.pyplot as plt
        %matplotlib inline
        import seaborn as sns
        sns.set()
        vector_file = '/Users/sbslee/Desktop/dokdo/data/moving-pictures-tutorial/faith_pd_vector.qza'
        metadata_file = '/Users/sbslee/Desktop/dokdo/data/moving-pictures-tutorial/sample-metadata.tsv'
        ax = dokdo.alpha_diversity_plot(vector_file, metadata_file, 'body-site', figsize=(8, 5))
        dokdo.addsig(0, 1, 20, t='***', ax=ax)
        dokdo.addsig(1, 2, 26, t='ns', ax=ax)
        plt.tight_layout()

    .. image:: images/addsig.png
    """
    if ax is None:
        fig, ax = plt.subplots(figsize=figsize)
    ax.plot([x1, x1, x2, x2], [y, y+h, y+h, y], lw=lw, c=lc)
    ax.text((x1+x2)*0.5, y+h, t, ha='center', va='bottom',
            color=tc, fontsize=fontsize)
    return ax


from selenium.webdriver.common.by import By

from PageObjects.HomePage0 import HomePage

#########################original imports###########################

#from pageObjects.CheckoutPage import CheckOutPage
# from pageObjects.HomePage import HomePage
from utilities.BaseClass import BaseClass

import time
###############################################################END#########################################

# Child Class inheritance Concept applied here
# @pytest.mark.usefixtures("setup")
# from selenium.webdriver.support.wait import WebDriverWait
# from selenium.webdriver.support.wait import WebDriverWait
# from PageObjects.CheckoutPage import CheckOutPage

# class TestOne(BaseClass):

# def test_e2e(self):
# homePage = HomePage(self.driver)
# checkoutpage = homePage.shopItems()
# Click is  handled in the line after this for Checkout page
# Nomore required the checkOutPage
# checkOutPage = CheckOutPage(self.driver)
# checkOutPage = homePage.shopItems()
# .click()
# checkOutPage = CheckOutPage(self.driver)
# cards = checkoutpage.getCardTitles()
# cards = self.driver.find_elements(By.CSS_SELECTOR,".card-title a")
# i = -1
# for card in cards:
#    i = i + 1
#   cardText = card.text
#  print(cardText)
#   if cardText == "Blackberry":
#      checkoutpage.getCardFooter()[i].click()

# self.driver(By.CSS_SELECTOR,".card-footer button")
# self.driver.find_element(By.CSS_SELECTOR("a[class*='btn-primary']")).click()
# confirmPage = checkoutpage.checkOutItems()
# .click()
# self.driver.find_element(By.XPATH, "//button[@class = 'btn btn-success']").click()
# self.driver.find_element(By.ID, "country").send_keys("ind")
# time.sleep
# self.verifyLinkPresence("India")

# self.driver.find_element(By.LINK_TEXT, "India").click()
# self.driver.find_element(By.XPATH, "//div[@class='checkbox checkbox-primary']").click()
# self.driver.find_element(By.CSS_SELECTOR, "[type='submit']").click()
# textMatch = self.driver.find_element(By.CSS_SELECTOR, "[class='alert-success']").text
# assert ("Success! Thank you!" in textMatch)

#########################original codes from udemy###########################################################################################################################

class TestOne(BaseClass):

    def test_e2e(self):
        log = self.getLogger()
        homePage = HomePage(self.driver)
        checkoutpage = homePage.shopItems()
        log.info("getting all the card titles")
        cards = checkoutpage.getCardTitles()
        i = -1
        for card in cards:
            i = i + 1
            cardText = card.text
            log.info(cardText)
            if cardText == "Blackberry":
                checkoutpage.getCardFooter()[i].click()

        self.driver.find_element(By.CSS_SELECTOR,"a[class*='btn-primary']").click()

        confirmpage = checkoutpage.checkOutItems()
        log.info("Entering country name as ind")
        self.driver.find_element(By.ID,"country").send_keys("ind")
        # time.sleep(5)
        self.verifyLinkPresence("India")
        #time.sleep(2)

        self.driver.find_element(By.LINK_TEXT,'India').click()
        self.driver.find_element(By.XPATH,"//div[@class='checkbox checkbox-primary']").click()
        # self.driver.find_element_by_xpath("//input[@type='submit']").click()
        self.driver.find_element(By.XPATH ,"//input[@type='submit']").click()
        #self.driver.find_element_by_css_selector("[type='submit']").click()
        textMatch = self.driver.find_element(By.CSS_SELECTOR,"[class*='alert-success']").text

        log.info("Text received from application is " + textMatch)

        assert ("Success! Thank you!" in textMatch)


import requests
from collections import deque
from lxml import etree
import hashlib
from pybloom_live import BloomFilter
import threading
import time
import os

class CrawlBSF:
    request_headers = {
        'host': "www.mafengwo.cn",
        'connection': "keep-alive",
        'cache-control': "no-cache",
        'upgrade-insecure-requests': "1",
        'user-agent': "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/55.0.2883.95 Safari/537.36",
        'accept': "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8",
        'accept-language': "zh-CN,en-US;q=0.8,en;q=0.6"
    }

    cur_level = 0
    max_level = 5
    dir_name = 'mafengwo/'
    iter_width = 50
    downloaded_urls = []
    if not os.path.exists(dir_name):
        os.makedirs(dir_name)
    du_md5_file_name = dir_name + 'md5.txt'  # save records of downloaded urls' md5
    du_url_file_name = dir_name + 'urls.txt'  # save records of downloaded urls

    bloom_downloaded_urls_md5 = BloomFilter(1024 * 1024 * 16, 0.01)
    bloom_url_queue = BloomFilter(1024 * 1024 * 16, 0.01)

    cur_queue = deque()  # current crawling quque
    child_queue = deque()  # storage quque for next level crawl

    def __init__(self, url):
        self.root_url = url
        self.cur_queue.append(url)
        self.du_file = open(self.du_url_file_name, 'a+')
        self.nm_url=0
        try:
            self.md5_file = open(self.du_md5_file_name, 'r')
            self.downloaded_urls = self.md5_file.readlines()
            self.md5_file.close()
            for urlmd5 in self.downloaded_urls:
                self.bloom_downloaded_urls_md5.add(urlmd5[:-2])
        except IOError:
            print("File not found")
        finally:
            self.md5_file = open(self.du_md5_file_name, 'a+')

    def enqueueUrl(self, url):
        if url not in self.bloom_url_queue and hashlib.md5(url.encode("utf-8")).hexdigest() not in crawler.bloom_downloaded_urls_md5:
            self.child_queue.append(url)
            self.bloom_url_queue.add(url)
            self.nm_url+=1

    def dequeuUrl(self):
        try:
            return self.cur_queue.popleft()
        except IndexError:
            return None

    def close(self):
        self.md5_file.close()
        self.du_file.close()

num_downloaded_pages = 0

# download the page content
def get_page_content(cur_url):
    global num_downloaded_pages
    print("downloading %s at level %d" % (cur_url, crawler.cur_level))
    try:
        req = requests.request("get", cur_url, headers=crawler.request_headers)
        req.raise_for_status()
        req.encoding = req.apparent_encoding
        html_page = req.text
        filename = cur_url[7:].replace('/', '_')
        with open("%s%s.html" % (crawler.dir_name, filename), 'wb+') as fo:
            fo.write(html_page.encode("utf-8"))
    except requests.RequestException as e:
        print(e)
        return
    except IOError as Arguments:
        print(Arguments)
        return
    except Exception as Arguments:
        print(Arguments)
        return
    # print 'add ' + hashlib.md5(cur_url).hexdigest() + ' to list'

    # save page and set bloomfilter
    dumd5 = hashlib.md5(cur_url.encode("utf-8")).hexdigest()
    crawler.bloom_downloaded_urls_md5.add(dumd5)
    crawler.md5_file.write(dumd5 + '\r\n')
    crawler.du_file.write(cur_url + '\r\n')
    crawler.bloom_downloaded_urls_md5.add(dumd5)
    num_downloaded_pages += 1

    html = etree.HTML(html_page.lower())
    hrefs = html.xpath(u"//a")
    for href in hrefs:
        try:
            if 'href' in href.attrib:
                val = href.attrib['href']
                if val.find('javascript:') != -1:
                    continue
                if val.startswith('http://') is False:
                    if val.startswith('/'):
                        val = 'http://www.mafengwo.cn' + val
                    else:
                        continue
                if val[-1] == '/':
                    val = val[0:-1]
                # if hashlib.md5(val).hexdigest() not in self.downloaded_urls:
                crawler.enqueueUrl(val)

                # else:
                # print 'Skip %s' % (val)
        except ValueError:
            continue

crawler = CrawlBSF("http://www.mafengwo.cn")
start_time = time.time()

# if it's the first page (start url), if true, crawl it in main thread in sync(blocking) mode
# 如果是第一个抓取页面的话，在主线程用同步（阻塞）的模式下载，后续的页面会通过创建子线程的方式异步爬取
is_root_page = True
threads = []
max_threads = 10

CRAWL_DELAY = 0.6

while True:
    url = crawler.dequeuUrl()
    # Go on next level, before that, needs to wait all current level crawling done
    if url is None:
        crawler.cur_level += 1
        for t in threads:
            t.join()
        if crawler.cur_level == crawler.max_level:
            break
        if len(crawler.child_queue) == 0:
            break

        print("第{}级共有{}个url，爬取中.....".format(
            crawler.cur_level,crawler.nm_url))
        crawler.cur_queue = crawler.child_queue
        crawler.child_queue = deque()
        crawler.nm_url=0
        continue

    # looking for an empty thread from pool to crawl

    if is_root_page:
        get_page_content(url)
        is_root_page = False
    else:
        while True:
            # first remove all finished running threads
            for t in threads:
                if not t.is_alive():
                    threads.remove(t)
            if len(threads) >= max_threads:
                time.sleep(CRAWL_DELAY)
                continue
            try:
                t = threading.Thread(
                    target=get_page_content, name=None, args=(url,))
                threads.append(t)
                # set daemon so main thread can exit when receives ctrl-c
                t.setDaemon(True)
                t.start()
                time.sleep(CRAWL_DELAY)
                break
            except Exception as e:
                print("Error: unable to start thread"+"%s" % e)
crawler.close()
print('%d pages downloaded, time cost %0.2f seconds' %
      (num_downloaded_pages, time.time()-start_time))


from Args import parse_args
from daisy.utils.loader import load_rate, get_ur, build_candidates_set, load_attack_data_get_attack_instances
from daisy.utils.map import neaten_id, get_unpopular, find_neighbor_items
from daisy.utils.splitter import split_test
from daisy.utils.sampler import Sampler
from daisy.utils.data import PairData, PointData
from daisy.utils.metrics import precision_at_k, recall_at_k, map_at_k, hr_at_k, ndcg_at_k, mrr_at_k
import random
import torch
import os
import numpy as np
import copy
import json
import pickle
import time
from tqdm import tqdm
import pandas as pd

def set_global_seeds(i):
    np.random.seed(i)
    random.seed(i)
    torch.manual_seed(i)
    torch.cuda.manual_seed(i)

class Env():
    def __init__(self, args):
        # set_global_seeds(28)
        self.args = args
        self.device = torch.device('cuda:{}'.format(self.args.gpu) if torch.cuda.is_available() else "cpu" )
        self._data_path = f'data/ds/{self.args.dataset}'
        self.save_path = f'{self.args.processed_path}_{self.args.dataset}'
        if not os.path.exists(self.save_path):
            os.makedirs(self.save_path)
        # attack parameters
        self.episode_length = self.args.episode_length
        # preprocess dataset
        df, self.user_num, self.item_num = load_rate(self.args.dataset, self.args.prepro, binary=True, level=self.args.level)
        # clean id + KG
        df, self.item_num_in_kg, self.n_entity, self.n_relation, self.kg, self.kg_dict, self.adj_entity, self.adj_relation, self.degree_item = neaten_id(self.args.dataset, df, self.args.kg_neighbor_size)
        self.degree = list(dict(sorted(self.degree_item.items(), key=lambda item: item[1],reverse=True)).keys())
        self.large_degree_items = self.degree[:int(len(self.degree)*0.3)]
        # generate KGE
        KGFILE = {'ml-1m':'ml1m-kg1m','ml-20m':'ml20m-kg500k','bx':'bx-kg150k','lastfm':'lastfm-kg15k'}
        if not os.path.exists(f'data/kg/{KGFILE[args.dataset]}/ent_embedding.pt'):
            print('============KGE============')
            kg_df = pd.DataFrame(self.kg, columns=['from', 'rel', 'to'])
            kg_df = kg_df.reindex(columns = ['from','to','rel'])
            self.train_KGE(kg_df,f'data/kg/{KGFILE[args.dataset]}/ent_embedding.pt')

        # print('len of rating',len(df))
        # Unpopular + popular
        unpopular_items,self.popular_items = get_unpopular(df, self.item_num_in_kg, args.unpop_ratio)
        # item pool
        self.item_pool = set(range(self.item_num))
        # target_item
        self.target_items = random.sample(unpopular_items, self.args.num_target_item)
        print(f'target items = {self.target_items}')
        # adj_item_dict
        self.adj_item_dict = find_neighbor_items(f'{self._data_path}/adj_item_{self.args.prepro}_{self.args.level}_{self.args.kg_hop}.pkl', self.adj_entity, self.item_num_in_kg, self.args.kg_hop)
        # Negative sampler
        self.spler = Sampler(
            self.user_num+self.args.num_max_attacker,
            self.item_num,
            num_ng=self.args.num_ng,
            sample_method=self.args.sample_method,
            sample_ratio=self.args.sample_ratio
        )
        # Train, Val, Test
        # self.processed_path = f'{self.save_path}/'
        # if not os.path.exists(self.processed_path):
        #     os.makedirs(self.processed_path)
        if os.path.exists(f'{self.save_path}/split_{self.args.num_ng}_{self.args.algo_name}_seed{self.args.seed}.pkl'):
            load_time = time.time()
            with open(f'{self.save_path}/split_{self.args.num_ng}_{self.args.algo_name}_seed{self.args.seed}.pkl','rb') as f:
                process_dict = pickle.load(f)
            self.train_set = process_dict['train_set']
            self.test_set = process_dict['test_set']
            self.test_ur = process_dict['test_ur']
            self.total_train_ur = process_dict['total_train_ur']
            self.test_ucands = process_dict['test_ucands']
            self.train_spy = process_dict['train_spy']
            self.eval_spy = process_dict['eval_spy']
            self.train_neg = process_dict['train_neg']
            self.test_neg = process_dict['test_neg']
            print(f'Load time elapsed= {time.time() - load_time}')
        else:
            self.train_set, self.test_set = split_test(df, 'ufo', self.args.test_size)
            # self.train_set, self.val_set = split_test(self.train_set, self.args.test_method, 0.1)
            self.test_ur = get_ur(self.test_set)
            self.total_train_ur = get_ur(self.train_set)
            self.test_ucands, self.train_spy, self.eval_spy = build_candidates_set(self.test_ur, self.total_train_ur, self.item_pool, self.args, self.target_items)
            self.train_neg = self.spler.transform(self.train_set, is_training=True)
            self.test_neg = self.spler.transform(self.test_set, is_training=True)
            # save!!
            save_dict = {
                'train_set':self.train_set,
                'test_set':self.test_set,
                'test_ur':self.test_ur,
                'total_train_ur':self.total_train_ur,
                'test_ucands':self.test_ucands,
                'train_spy':self.train_spy,
                'eval_spy':self.eval_spy,
                'train_neg':self.train_neg,
                'test_neg':self.test_neg
            }
            with open(f'{self.save_path}/split_{self.args.num_ng}_{self.args.algo_name}_seed{self.args.seed}.pkl', 'wb') as f:
                pickle.dump(save_dict,f)

        # model
        self.build_model()

    def build_model(self):
        self.result_save_path = f'{self.save_path}/{self.args.algo_name}'+\
                f'{self.args.prepro}_{self.args.level}_{self.args.test_method}_{self.args.epochs}_{self.args.sample_ratio}_{self.args.sample_method}_{self.args.loss_type}_{self.args.problem_type}'+\
                    f'{self.args.dataset}'
        if self.args.problem_type == 'pair':
            train_dataset = PairData(self.train_neg, is_training=True)
        else:
            train_dataset = PointData(self.train_neg, is_training=True)
        if self.args.problem_type == 'pair':
            if self.args.algo_name == 'NeuMF':
                from daisy.model.pair.NeuMFRecommender import PairNeuMF
                print(f'================tareget_model [{self.args.algo_name}_{self.args.problem_type}]===============')
                self.model = PairNeuMF(
                    self.user_num+self.args.num_max_attacker,
                    self.item_num,
                    factors=self.args.factors,
                    num_layers=self.args.num_layers,
                    q=self.args.dropout,
                    lr=self.args.lr,
                    epochs=self.args.epochs,
                    reg_1=self.args.reg_1,
                    reg_2=self.args.reg_2,
                    loss_type=self.args.loss_type,
                    gpuid=self.device
                )
        elif self.args.problem_type == 'point':
            if self.args.algo_name == 'NeuMF':
                from daisy.model.point.NeuMFRecommender import PointNeuMF
                print(f'================tareget_model [{self.args.algo_name}_{self.args.problem_type}]===============')
                self.model = PointNeuMF(
                    self.user_num+self.args.num_max_attacker,
                    self.item_num,
                    factors=self.args.factors,
                    num_layers=self.args.num_layers,
                    q=self.args.dropout,
                    lr=self.args.lr,
                    epochs=self.args.epochs,
                    reg_1=self.args.reg_1,
                    reg_2=self.args.reg_2,
                    loss_type=self.args.loss_type,
                    gpuid=self.device
                )
        # 如果有模型，就加载模型
        if os.path.exists(self.result_save_path+f'/model_{self.args.factors}_{self.args.num_layers}_{self.args.dropout}_{self.args.epochs}_{self.args.reg_1}_{self.args.reg_2}.pth'):
            self.model.load_state_dict(torch.load(self.result_save_path+f'/model_{self.args.factors}_{self.args.num_layers}_{self.args.dropout}_{self.args.epochs}_{self.args.reg_1}_{self.args.reg_2}.pth',map_location='cuda:0'))
            self.model.to(self.device)
            # 构造test_ucands
            # model.evaluate_model()
        # 否则，走一遍模型的全流程(earlystop, checkpoint)
        else:
            train_loader = torch.utils.data.DataLoader(
                train_dataset,
                batch_size=128,
                shuffle=True,
                num_workers=4
            )
            if not os.path.exists(self.result_save_path):
                os.makedirs(self.result_save_path)
            s_time = time.time()
            self.model.fit(train_loader)
            elapsed_time = time.time() - s_time
            print(f'prepare the black box model elapsed_time= {elapsed_time}')
            torch.save(self.model.state_dict(), self.result_save_path+f'/model_{self.args.factors}_{self.args.num_layers}_{self.args.dropout}_{self.args.epochs}_{self.args.reg_1}_{self.args.reg_2}.pth')

        #res = self.evaluate(self.test_ucands, self.test_ur)
        #print(res)
        # print('===========before attack=========')
        # for i in self.target_items:
        #     res = self.evaluate(self.train_spy, target_item = i)
        #     print(f'target [{i}] , [{res}]')

    def reset_model(self):
        if os.path.exists(self.result_save_path):
            self.model.load_state_dict(torch.load(self.result_save_path+'/model.pth'))
            self.model.to(self.device)
    def get_target_items(self):
        return self.target_items

    def get_meta(self):
        return self.user_num, self.item_num, self.item_num_in_kg, self.target_items, self.kg,\
         self.n_entity, self.n_relation, self.adj_entity, self.adj_relation,self.adj_item_dict,self.popular_items,self.large_degree_items

    def attack(self, attack_data, target_item, eval=False):
        # attack_data to df
        print('====convert attack profiles to attack df====')
        ui, user, item, r, t = {},[],[],[],[]
        for p in attack_data:
            u = p[0]
            for i in p[1:]:
                user.append(u)
                item.append(i)
                r.append(1.0)
                t.append(1)
        ui['user'] = user
        ui['item'] = item
        ui['rating'] = r
        ui['timestamp'] = t
        attack_df = pd.DataFrame(ui)
        # get train set
        attack_neg_set = self.spler.transform(attack_df, is_training=True)
        if self.args.problem_type == 'pair':
            attack_dataset = PairData(attack_neg_set, is_training=True)
        else:
            attack_dataset = PointData(attack_neg_set, is_training=True)

        attack_loader = torch.utils.data.DataLoader(
                attack_dataset,
                batch_size=128,
                shuffle=True,
                num_workers=4
            )
        self.model.set_epoch(self.args.attack_epochs)
        self.model.fit(attack_loader)

        # evaluate

        res_train = self.evaluate(self.train_spy, target_item = target_item)
        #print(f'evaluate train spy hr [{hr_train}] ndcg [{ndcg_train}]')
        res_val = self.evaluate(self.eval_spy, target_item = target_item)
        #print(f'evaluate eval spy hr [{hr_eval}] ndcg [{ndcg_eval}]')
        return res_train, res_val

    def evaluate(self, test_ucands, test_ur = None, target_item=None):
        assert test_ur is not None or target_item is not None
        # print(f'Start Calculating Metrics......each user have [{self.args.candi_num}] candidates')
        preds = {}
        for u in tqdm(test_ucands.keys()):
            # truncate the acutual candidates num
            if target_item is None:
                test_ucands[u] = test_ucands[u][:self.args.candi_num]
            else:
                test_ucands[u] = test_ucands[u][:self.args.candi_num]+[target_item]
            # build a test MF dataset for certain user u to accelerate
            tmp = pd.DataFrame({
                'user': [u for _ in test_ucands[u]],
                'item': test_ucands[u],
                'rating': [0. for _ in test_ucands[u]], # fake label, make nonsense
            })
            tmp_neg_set = self.spler.transform(tmp, is_training=False)
            if self.args.problem_type == 'pair':
                tmp_dataset = PairData(tmp_neg_set, is_training=False)
            else:
                tmp_dataset = PointData(tmp_neg_set, is_training=False)
            tmp_loader = torch.utils.data.DataLoader(
                tmp_dataset,
                batch_size=128,
                shuffle=False,
                num_workers=0
            )
            topk_indices = self.model.evaluate_model(tmp_loader, self.args.topk)
            preds[u] = torch.take(torch.tensor(test_ucands[u]), topk_indices).cpu().numpy()
        for u in preds.keys():
            if target_item is None:
                preds[u] = [1 if i in test_ur[u] else 0 for i in preds[u]]
            else:
                preds[u] = [1 if i==target_item else 0 for i in preds[u]]
            # process topN list and store result for reporting KPI
        print('Save metric@k result to res folder...')
        res = pd.DataFrame({'metric@K': ['hr', 'ndcg']})
        hr, ndcg = 0, 0
        for k in [1, 5, 10, 20, 30, 50]:
            if k > self.args.attack_topk:
                continue
            tmp_preds = preds.copy()
            tmp_preds = {key: rank_list[:k] for key, rank_list in tmp_preds.items()}
            hr_k = hr_at_k(tmp_preds)
            ndcg_k = np.mean([ndcg_at_k(r, k) for r in tmp_preds.values()])
            # if k == 10:
            #     print(f'HR@{k}: {hr_k:.4f}')
            #     print(f'NDCG@{k}: {ndcg_k:.4f}')

            res[k] = np.array([hr_k, ndcg_k])
            # if k == self.args.attack_topk:
            #     hr, ndcg = hr_k, ndcg_k
        # res.to_csv(
        #     f'{self.result_save_path}/kpi_results.csv',
        #     index=False
        # )
        return res

    def candidate(self, agent_pivot = None):
        # generate valid action set
        candis = []
        # mask = np.array(mask).T

        for b_idx in range(len(self.state)):
            # m = mask[b_idx]
            if self.args.candidate_mode == 'nearest_neighbor':
                pivot = self.state[b_idx][self.step_count]
                pivot_neighbor = self.adj_item_dict[pivot]
                if len(pivot_neighbor) >= self.args.action_size:
                    candis.append(random.sample(pivot_neighbor, self.args.action_size))
                else:
                    rr = self.args.action_size - len(pivot_neighbor)
                    candi = random.sample(list(range(self.item_num_in_kg)), rr) + pivot_neighbor
                    candis.append(candi)
            elif self.args.candidate_mode == 'kg_item':
                candi = random.sample(list(range(self.item_num_in_kg)), self.args.action_size)
                candis.append(candi)
            elif self.args.candidate_mode == 'all_item':
                candi = random.sample(list(range(self.item_num)), self.args.action_size)
                candis.append(candi)
            elif self.args.candidate_mode == 'target_neighbor':
                pivot = self.state[b_idx][1]
                pivot_neighbor = self.adj_item_dict[pivot]
                if len(pivot_neighbor) >= self.args.action_size:
                    candis.append(random.sample(pivot_neighbor, self.args.action_size))
                else:
                    rr = self.args.action_size - len(pivot_neighbor)
                    candi = random.sample(list(range(self.item_num_in_kg)), rr) + pivot_neighbor
                    candis.append(candi)
            elif self.args.candidate_mode == 'agent_select':
                assert agent_pivot is not None
                pivot = self.state[b_idx][agent_pivot[b_idx]]
                pivot_neighbor = self.adj_item_dict[pivot]
                if len(pivot_neighbor) >= self.args.action_size:
                    candis.append(random.sample(pivot_neighbor, self.args.action_size))
                else:
                    rr = self.args.action_size - len(pivot_neighbor)
                    candi = random.sample(list(range(self.item_num_in_kg)), rr) + pivot_neighbor
                    candis.append(candi)
            # candi = set(tmp) - set(m)
        return np.array(candis)
    def p_mask(self):
        if random.random() < self.args.bandit_ratio:
            mask = [[True]*(self.step_count+1) + [False]*(self.episode_length-self.step_count-1)]*len(self.attackers_id)
        else:
            mask = [[True] + [False]*(self.episode_length-1)]*len(self.attackers_id)
        return mask
    '''Gym environment API'''
    def reset(self, target_item, episode):
        self.target_item = target_item
        self.attackers_id = [i for i in range(self.user_num+episode*self.args.num_train_attacker, self.user_num+(episode+1)*self.args.num_train_attacker)]
        self.init_item = [target_item for _ in range(len(self.attackers_id))]
        self.state = np.hstack((np.array(self.attackers_id)[:, None], np.array(self.init_item)[:,None])) # (num_train_attacker,2)
        # Pad state to keep dimension consistent
        self.state = np.pad(self.state, ((0,0),(0,self.episode_length - 1)), constant_values = -1)
        self.step_count = 1
        # State dict
        print(f'action candidates mode [{self.args.candidate_mode}], action size = [{self.args.action_size}]')

        if self.args.candidate_mode == 'agent_select':
            agent_pivot = [1 for _ in range(len(self.attackers_id))] # target item as the initial pivot
            self.valid_actions = self.candidate(agent_pivot) # the neighbors of pivot as the valid actions

            self.pivot_mask = self.p_mask() # batch_size * mask. the target_item and the second item could be the pivot
            return {'some_state':torch.LongTensor(copy.deepcopy(self.state)),
                 'valid_actions': torch.LongTensor(copy.deepcopy(self.valid_actions)),
                    'pivot_mask': torch.LongTensor(copy.deepcopy(self.pivot_mask))
                    }
        else:
            self.valid_actions = self.candidate()
            return {'some_state':torch.LongTensor(copy.deepcopy(self.state)),
                 'valid_actions': torch.LongTensor(copy.deepcopy(self.valid_actions))
                    }
    '''Gym environment API'''
    def step(self, item_action, pivot_action):
        '''
        input:
            action_chosen: [B] 每一行都有一个新的action(item id)
        return
            reward: [bs*attack_user_num] 到了episode长度时，attack_user_num个profile共同得到一个reward，一共有bs个reward，并需要normalize处理，然后repeat
            done: 到了episode长度时，返回True，否则返回False
        '''
        self.step_count += 1
        if pivot_action is None:
            action = self.valid_actions[range(self.valid_actions.shape[0]), item_action.cpu().numpy()]
            self.state[:,self.step_count] = action
            self.valid_actions = self.candidate()
        else:
            # print('pivot_action',pivot_action)
            action = self.valid_actions[range(self.valid_actions.shape[0]), item_action.cpu().numpy()]
            self.state[:,self.step_count] = action # update state must before self.candidate(pivot_action)
            self.valid_actions = self.candidate(pivot_action.cpu().numpy()+1)
            # the pivot mask only affect when mode == 'agent_pivot'
            self.pivot_mask = self.p_mask()

        if self.step_count == self.episode_length:
            res_train, res_val = self.attack(self.state, self.target_item)
            if self.args.rl_reward_type == 'hr':
                rew = res_train.loc[0,self.args.reward_topk]
            elif self.args.rl_reward_type == 'ndcg':
                rew = res_train.loc[1,self.args.reward_topk]
            user_rewards = []
            user_rewards.extend([rew for j in range(len(self.attackers_id))])
            reward = np.array(user_rewards)
            done = [True]*len(self.attackers_id)
        else:
            res_train = -1
            res_val = -1
            reward = np.zeros(len(self.attackers_id))
            done = [False]*len(self.attackers_id)

        info = {'res_train':res_train, 'res_val':res_val }
        if pivot_action is None:
            return {'some_state':torch.LongTensor(copy.deepcopy(self.state)),
                    'valid_actions': torch.LongTensor(copy.deepcopy(self.valid_actions))
                    }, reward, done, info
        else:
            return {'some_state':torch.LongTensor(copy.deepcopy(self.state)),
                    'valid_actions': torch.LongTensor(copy.deepcopy(self.valid_actions)),
                    'pivot_mask': torch.LongTensor(copy.deepcopy(self.pivot_mask))
                    }, reward, done, info
if __name__=='__main__':
    from collections import defaultdict
    target_items_results_hr_5 = defaultdict(list)
    target_items_results_hr_10 = defaultdict(list)
    target_items_results_hr_20 = defaultdict(list)
    target_items_results_ndcg_5 = defaultdict(list)
    target_items_results_ndcg_10 = defaultdict(list)
    target_items_results_ndcg_20 = defaultdict(list)

    args = parse_args()
    set_global_seeds(args.seed)
    env = Env(args)
    target_items = env.get_target_items()
    print('===========target_items==========')
    print(target_items)
    n_users, n_items, _, _, _, _,_,_,_, adj_item_dict, popular_items, large_degree_items= env.get_meta()
    episode_length = args.episode_length # 16:hr0.32, 32:hr0.42, 64:hr0.9
    eval_n_attacker = 3
    sample_times = 50
    alpha = args.action_size    # 1000 ok 500变弱， 1500变弱

    if args.random_method == 'n':
        method = 'target_neighbor_attack'
    elif args.random_method == 'r':
        method = 'target_random_attack'
    elif args.random_method == 'rn':
        method = 'neighbor_random_attack'
    elif args.random_method == 'd':
        method = 'degree_attack'
    for tar in range(len(target_items)):
        set_global_seeds(args.seed)
        env = Env(args)
        # if tar in [0,1,2,3,4,5,6]: continue
        target_item = target_items[tar]
        # print(f'target_item is {target_item}')
        # print('*'*50)
        num_users = n_users
        pivot = target_item
        # env.reset_model()
        final_results_train = {'hr':{5:[],10:[],20:[]}, 'ndcg':{5:[],10:[],20:[]}}
        final_results_val = {'hr':{5:[],10:[],20:[]}, 'ndcg':{5:[],10:[],20:[]}}
        # item_set_random = random.sample(list(range(n_items)),alpha)
        for s in range(sample_times):
            attack_data = []
            for i in tqdm(range(num_users, num_users+eval_n_attacker),desc='num attacker'):
                attack_profile = [i]
                if random.random() < args.target_bandit:
                    attack_profile.append(target_item)
                else:
                    attack_profile.append(random.choice(list(range(n_items))))

                while(len(attack_profile)-1<=episode_length):
                    # item_set = random.sample(list(range(n_items)),alpha)
                    if args.random_method == 'n':
                        pivot_list = adj_item_dict[pivot]
                        # item_set = random.sample(pivot_list,alpha)
                        item_set = pivot_list
                        j2 = random.choice(item_set)
                        # pivot = j2
                    elif args.random_method == 'r':
                        item_set_random = list(range(n_items))
                        j2 = random.choice(item_set_random)
                    elif args.random_method == 'p':
                        item_set = popular_items
                        j2 = random.choice(item_set)
                    elif args.random_method == 'd':
                        if alpha<=len(large_degree_items):
                            print('no plus')
                            item_set = random.sample(large_degree_items,alpha)
                        else:
                            print('no plus')
                            item_set = random.sample(large_degree_items,alpha - len(large_degree_items)) + large_degree_items
                        # item_set = random.sample(list(range(n_items)),alpha) + large_degree_items
                        j2 = random.choice(item_set)
                    else:
                        if alpha<=len(adj_item_dict[pivot]):
                            print('no plus')
                            item_set = random.sample(adj_item_dict[pivot],alpha)
                        else:
                            print('no plus')
                            item_set = random.sample(list(range(n_items)),alpha - len(adj_item_dict[pivot])) + adj_item_dict[pivot]
                        # item_set = random.sample(list(range(n_items)),alpha) + adj_item_dict[pivot]
                        j2 = random.choice(item_set)
                    attack_profile.append(j2)
                    # item_set.remove(j2)
                attack_data.append(attack_profile)
            res_train, res_val = env.attack(attack_data, target_item)
            for k in [5, 10, 20]:
                final_results_train['hr'][k].append(res_train.loc[0,k])
                final_results_train['ndcg'][k].append(res_train.loc[1,k])
                final_results_val['hr'][k].append(res_val.loc[0,k])
                final_results_val['ndcg'][k].append(res_val.loc[1,k])
            num_users = num_users+eval_n_attacker
        # print(result_dict)
        target_items_results_hr_5[target_item] = final_results_val['hr'][5]
        target_items_results_ndcg_5[target_item] = final_results_val['ndcg'][5]
        target_items_results_hr_10[target_item] = final_results_val['hr'][10]
        target_items_results_ndcg_10[target_item] = final_results_val['ndcg'][10]
        target_items_results_hr_20[target_item] = final_results_val['hr'][20]
        target_items_results_ndcg_20[target_item] = final_results_val['ndcg'][20]
        del env
    # HR
    target_items_results_hr5_df = pd.DataFrame(target_items_results_hr_5)
    target_items_results_hr5_df[-1] = target_items_results_hr5_df.mean(1)
    target_items_results_hr10_df = pd.DataFrame(target_items_results_hr_10)
    target_items_results_hr10_df[-1] = target_items_results_hr10_df.mean(1)
    target_items_results_hr20_df = pd.DataFrame(target_items_results_hr_20)
    target_items_results_hr20_df[-1] = target_items_results_hr20_df.mean(1)
    # NDCG
    target_items_results_ndcg_5 = pd.DataFrame(target_items_results_ndcg_5)
    target_items_results_ndcg_5[-1] = target_items_results_ndcg_5.mean(1)
    target_items_results_ndcg_10 = pd.DataFrame(target_items_results_ndcg_10)
    target_items_results_ndcg_10[-1] = target_items_results_ndcg_10.mean(1)
    target_items_results_ndcg_20 = pd.DataFrame(target_items_results_ndcg_20)
    target_items_results_ndcg_20[-1] = target_items_results_ndcg_20.mean(1)
    print(f'============avg of each iteration method {args.random_method} action_size {args.action_size} seed {args.seed}============')
    print('top20')
    print(target_items_results_hr20_df[-1])
    print('top10')
    print(target_items_results_hr10_df[-1])
    print('top5')
    print(target_items_results_hr5_df[-1])
    print('ndcg20')
    print(target_items_results_ndcg_20[-1])
    print('ndcg10')
    print(target_items_results_ndcg_10[-1])
    print('ndcg5')
    print(target_items_results_ndcg_5[-1])

    if args.random_method == 'r':
        target_items_results_df.to_csv(f'result/{method}/seed_{args.seed}_epi{episode_length}_att{eval_n_attacker}_dim{args.init_dim}{args.dim1}{args.dim2}_kghop{args.kg_hop}.csv', index=False)
    else:
        target_items_results_df.to_csv(f'result/{method}/seed_{args.seed}_action_size_{args.action_size}_epi{episode_length}_att{eval_n_attacker}_dim{args.init_dim}{args.dim1}{args.dim2}_kghop{args.kg_hop}.csv', index=False)
    # kg neighbor attack


import pandas as pd
import random
from pykrige.rk import RegressionKriging
from sklearn import linear_model, naive_bayes, gaussian_process, model_selection, metrics

# Full Dataset
# crash = pd.read_msgpack('Crash_Data__1997_to_Current__Transportation.msg')

# Split to Fatal/Non-Fatal
# fatalFilter = crash['Fatality Count'] > 0
# crash[fatalFilter].to_csv('fatalCrashes.csv', index=False)
# crash[~fatalFilter].to_csv('nonFatalCrashes.csv', index=False)
# crash[fatalFilter].to_msgpack('fatalCrashes.msg')
# crash[~fatalFilter].to_msgpack('nonFatalCrashes.msg')

# Load Fatal/Non-Fatal
n_nonfatal = 2600409
n_fatal = 25852
skip = sorted(random.sample(xrange(n_nonfatal), (n_nonfatal - n_fatal)))
fatal = pd.read_csv('fatalCrashes.csv')
nonfatal = pd.read_csv('nonFatalCrashes.csv', skiprows=skip)
nonfatal.columns = list(fatal)
crash = pd.concat([fatal, nonfatal], axis=0)

# First 1000
# crash = pd.read_csv('Crash_Data__1997_to_Current__Transportation_5000.csv')
# Select Subset of Variables

######################################################################################################################################################
# Clean Data
######################################################################################################################################################

keepVar = [
    'Crash Record Number',
    'County Code',
    'Municipality Code',
    'District Code',
    'Police Agency Code',
    'Crash Month',
    'Day of Week',
    'Time of Day',
    'Illumination',
    'Weather',
    'Road Condition',
    'Collision Type',
    'Work Zone Type',
    'Work Zone Location',
    'Intersection Type',
    'Traffic Control Device',
    'Location Type',
    'Urban / Rural',
    'Fatality Count',
    # 'Injury Count',
    'Person Count',
    'Total Unit Count',
    'School Bus Indicator',
    'School Zone Indicator',
    'Dispatch Time',
    'Construction Zone Speed Limit',
    'Estimated Hours Closed',
    'Lane Closed',
    'Lane Closure Direction',
    'Notify Highway Maintenance',
    'Special Jurisdiction',
    'Traffic Control Device Function',
    'Traffic Detour',
    'Workers Present',
    'Work Zone Close/Detour',
    'Work Zone Flagger',
    'Work Zone Law Officer',
    'Work Zone Closure',
    'Work Zone Moving',
    'Work Zone Other',
    'Work Zone Shoulder/Median',
    'Vehicle Count',
    'Automobile Count',
    'Motorcycle Count',
    'Bus Count',
    'Small Truck Count',
    'Heavy Truck Count',
    'SUV Count',
    'Van Count',
    'Bicycle Count',
    'Suspected Serious Injury Count',
    'Suspected Minor Injury Count',
    'Possible Injury Count',
    'Total Injury Count',
    'Unknown Injury Degree Count',
    'Unknown Injury Person Count',
    '16 Year Old Driver Count',
    '17 Year Old Driver Count',
    '18 Year Old Driver Count',
    '19 Year Old Driver Count',
    '20 Year Old Driver Count',
    '50-64 Year Old Driver Count',
    '65-74 Year Old Driver Count',
    '75 Plus Year Old Driver Count',
    'Unbelted Occupant Count',
    'Unbelted Death Count',
    'Unbelted Suspected Serious Injury Count',
    'Belted Death Count',
    'Belted Suspected Serious Injury Count',
    'Motorcycle Death Count',
    'Motorcycle Suspected Serious Injury Count',
    'Bicycle Death Count',
    'Bicycle Suspected Serious Injury Count',
    'Pedestrian Count',
    'Pedestrian Death Count',
    'Pedestrian Suspected Serious Injury Count',
    'Maximum Severity Level',
    'Commercial Vehicle Count',
    'Latitude (Decimal)',
    'Longitude (Decimal)',
    'Roadway Surface Type',
    'Interstate',
    'State Road',
    'Local Road Only',
    'Turnpike',
    'Wet Road',
    'Snow Slush Road',
    'Icy Road',
    'Sudden Deer',
    'Shoulder Related',
    'Rear End',
    'Head On / Opposite Direction Side Swipe',
    'Hit Fixed Object',
    'Single Vehicle Run Off Road',
    'Work Zone',
    # 'Property Damage Only',
    # 'Injury or Fatal',
    # 'Fatal or Suspected Serious Injury',
    # 'Injury',
    # 'Fatal',
    'Non Intersection',
    'Intersection',
    'Signalized Intersection',
    'Stop Controlled Intersection',
    'Unsignalized Intersection',
    'School Bus',
    'School Zone',
    'Hit Deer',
    'Hit Tree / Shrub',
    'Hit Embankment',
    'Hit Pole',
    'Hit Guide Rail',
    'Hit Guide Rail End',
    'Hit Barrier',
    'Hit Bridge',
    'Overturned',
    'Motorcycle',
    'Bicycle',
    'Heavy Truck Related',
    'Vehicle Failure',
    'Train / Trolley',
    'Phantom Vehicle',
    'Alcohol Related',
    'Drinking Driver',
    'Underage Drinking Driver',
    'Unlicensed',
    'Distracted',
    'Cell Phone',
    'No Clearance',
    'Running Red Light',
    'Tailgating',
    'Cross Median',
    'Curved Road',
    'Curve Driver Error',
    'Limit 65 MPH',
    'Speeding',
    'Speeding Related',
    'Aggressive Driving',
    'Fatigue / Asleep',
    'Driver 16 Years Old',
    'Driver 17 Years Old',
    'Driver 18 Years Old',
    'Driver 19 Years Old',
    'Driver 20 Years Old',
    'Driver 50 - 64 Years Old',
    'Driver 65 - 74 Years Old',
    'Driver 75 Plus',
    'Unbelted',
    'Pedestrian',
    'Commercial Vehicle',
    'PSP Reported',
    'NHTSA Aggressive Driving',
    'Deer Related',
    'Illumination Dark',
    'Running Stop Sign',
    'Train',
    'Trolley',
    'Hit Parked Vehicle',
    'Fire In Vehicle',
    'Vehicle Towed',
    'Hazardous Truck',
    'Suspected Serious Injury',
    'Suspected Minor Injury',
    'Possible Injury',
    'Motorcycle Drinking Driver',
    'Drug Related',
    'Illegal Drug Related',
    'School Bus Unit',
    'Drugged Driver',
    'Impaired Driver'
    ]

crash = crash[keepVar]

convertBoolean = [
    'School Bus Indicator',
    'School Zone Indicator',
    'Interstate',
    'State Road',
    'Local Road Only',
    'Turnpike',
    'Wet Road',
    'Snow Slush Road',
    'Icy Road',
    'Sudden Deer',
    'Shoulder Related',
    'Rear End',
    'Head On / Opposite Direction Side Swipe',
    'Hit Fixed Object',
    'Single Vehicle Run Off Road',
    'Work Zone',
    'Property Damage Only',
    'Injury or Fatal',
    'Fatal or Suspected Serious Injury',
    'Injury',
    'Fatal',
    'Non Intersection',
    'Intersection',
    'Signalized Intersection',
    'Stop Controlled Intersection',
    'Unsignalized Intersection',
    'School Bus',
    'School Zone',
    'Hit Deer',
    'Hit Tree / Shrub',
    'Hit Embankment',
    'Hit Pole',
    'Hit Guide Rail',
    'Hit Guide Rail End',
    'Hit Barrier',
    'Hit Bridge',
    'Overturned',
    'Motorcycle',
    'Bicycle',
    'Heavy Truck Related',
    'Vehicle Failure',
    'Train / Trolley',
    'Phantom Vehicle',
    'Alcohol Related',
    'Drinking Driver',
    'Underage Drinking Driver',
    'Unlicensed',
    'Distracted',
    'Cell Phone',
    'No Clearance',
    'Running Red Light',
    'Tailgating',
    'Cross Median',
    'Curved Road',
    'Curve Driver Error',
    'Limit 65 MPH',
    'Speeding',
    'Speeding Related',
    'Aggressive Driving',
    'Fatigue / Asleep',
    'Driver 16 Years Old',
    'Driver 17 Years Old',
    'Driver 18 Years Old',
    'Driver 19 Years Old',
    'Driver 20 Years Old',
    'Driver 50 - 64 Years Old',
    'Driver 65 - 74 Years Old',
    'Driver 75 Plus',
    'Unbelted',
    'Pedestrian',
    'Commercial Vehicle',
    'PSP Reported',
    'NHTSA Aggressive Driving',
    'Deer Related',
    'Illumination Dark',
    'Running Stop Sign',
    'Train',
    'Trolley',
    'Hit Parked Vehicle',
    'Fire In Vehicle',
    'Vehicle Towed',
    'Hazardous Truck',
    'Suspected Serious Injury',
    'Suspected Minor Injury',
    'Possible Injury',
    'Motorcycle Drinking Driver',
    'Drug Related',
    'Illegal Drug Related',
    'School Bus Unit',
    'Drugged Driver',
    'Impaired Driver',
    'Lane Closed'
]

for col in convertBoolean:
    if col in keepVar:
        crash[col] = crash[col].map({'Yes': 1, 'No': 0})
        crash[col] = crash[col].fillna(0)

for col in ['Traffic Detour',
            'Workers Present',
            'Work Zone Close/Detour',
            'Work Zone Flagger',
            'Work Zone Law Officer',
            'Work Zone Closure',
            'Work Zone Moving',
            'Work Zone Other',
            'Work Zone Shoulder/Median']:
    if col in keepVar:
        crash[col] = crash[col].map({'y': 1, 'Y': 1, 'n': 0, 'N': 0})
        crash[col] = crash[col].fillna(0)

convertCategorical = [
    # 'County Code',
    # 'Municipality Code',
    # 'District Code',
    # 'Police Agency Code',
    'Crash Month',
    'Day of Week',
    'Illumination',
    'Weather',
    'Road Condition',
    'Collision Type',
    'Work Zone Type',
    'Work Zone Location',
    'Intersection Type',
    'Traffic Control Device',
    'Location Type',
    'Urban / Rural',
    'Lane Closure Direction',
    'Notify Highway Maintenance',
    'Special Jurisdiction',
    'Traffic Control Device Function',
    'Roadway Surface Type'
]

convertCategorical = [x for x in convertCategorical if x in keepVar]

crashCategorical = pd.get_dummies(crash[convertCategorical], dummy_na=True)
crash = pd.concat([crash.drop(convertCategorical, axis=1), crashCategorical], axis=1)

# Fix dangling cases
crash['Estimated Hours Closed'] = crash['Estimated Hours Closed'].replace(r'\s+', 0, regex=True)
crash['Estimated Hours Closed'] = pd.to_numeric(crash['Estimated Hours Closed'])
crash['Construction Zone Speed Limit'] = crash['Construction Zone Speed Limit'].apply(lambda x: 1 if x > 0 else 0)

######################################################################################################################################################
# Summarize
######################################################################################################################################################

# muni = pd.read_csv('Municipality_Boundary.csv')
# muni['Municipality Code'] = muni['FIPS_MUN_C']
# crash = pd.merge(crash, muni[['Municipality Code', 'FIPS_STATE']], on='Municipality Code', how='outer')
# crash = crash.drop(['FIPS_STATE', 'County Code', 'District Code', 'Police Agency Code', 'Crash Record Number'], axis=1).fillna(0)
# crash_muni = crash.groupby('Municipality Code').sum().reset_index()
# print(crash_muni)
# crash_muni.to_csv('crash_muni.csv')

######################################################################################################################################################
# First Pass
######################################################################################################################################################

# Drop any accidents without lat/lon
crash = crash.dropna(axis=0)

X = crash.drop(['Crash Record Number', 'Fatality Count', 'Latitude (Decimal)', 'Longitude (Decimal)',
                'Municipality Code', 'County Code', 'District Code', 'Police Agency Code'], axis=1).as_matrix()
coord = crash[['Latitude (Decimal)', 'Longitude (Decimal)']].as_matrix()
y = crash['Fatality Count'].as_matrix()

X_train, X_test, coord_train, coord_test, y_train, y_test = model_selection.train_test_split(X, coord, y, test_size=0.1, random_state=100)

######################################################################################################################################################
# Models
######################################################################################################################################################

# ols = linear_model.LinearRegression()
# ridge = linear_model.Ridge()
# grad = linear_model.SGDRegressor()
# nb = naive_bayes.MultinomialNB()
# gp = gaussian_process.GaussianProcessRegressor()

# models = [ols, ridge, grad, nb, gp]

# print('=' * 150)
# print('=' * 150)
# print('Simple Regression')
# print('=' * 150)

# for mod in models:
#     print('=' * 150)
#     print('regression model:', mod.__class__.__name__)
#     mod.fit(X_train, y_train)
#     print('MSE: ', metrics.mean_squared_error(y_test, mod.predict(X_test)))
#     print('R^2: ', metrics.r2_score(y_test, mod.predict(X_test)))

######################################################################################################################################################
# Spatial Autocorrelation
######################################################################################################################################################

ols = linear_model.LinearRegression()
ridge = linear_model.Ridge()
grad = linear_model.SGDRegressor()
gp = gaussian_process.GaussianProcessRegressor()

models = [ols, ridge, grad, gp]

print(coord)

print('=' * 150)
print('=' * 150)
print('Kriging Regression')
print('=' * 150)

for mod in models:
    print('=' * 150)
    print('regression model:', mod.__class__.__name__)
    modRK = RegressionKriging(regression_model=mod, n_closest_points=20)
    modRK.fit(X_train, coord_train, y_train)
    print('Regression Rsquared: ', modRK.regression_model.score(X_test, y_test))
    print('RK Rsquared: ', modRK.score(X_test, coord_test, y_test))


import sys
import re
import os
import hashlib
import time
import subprocess
import multiprocessing

# we define the a few global variables for constant values
DEFAULTHOST = 'tp-4b01-@@'
WORKERS_FILE = 'machines.txt'
USERNAME = 'dalbianco-20'
MAXRETRY = 5

# This function launches a sub process
def launch_subprocess(worker, command, hostname=None):
    # the command to be executed, contains two commands: one to create the remote dir and one to copy the files
    if command == 'mkdir':
        # we're using the -q otpion for quiet mode
        commands = ['ssh', '-q', USERNAME+'@'+worker,'mkdir -p /tmp/'+USERNAME+'/shufflesreceived']
    elif command == 'scp':
        commands = ['scp', '-qC', '/tmp/'+USERNAME+'/shuffles/'+worker+'-'+hostname+'.txt', USERNAME+'@'+worker+':/tmp/'+USERNAME+'/shufflesreceived/']

    # simply create a sub process and return it
    #print('{}: starting {} ...'.format(worker, command))
    process = subprocess.Popen(
                    commands,
                    stdout=subprocess.PIPE,
                    stderr=subprocess.PIPE,
                    stdin=subprocess.PIPE,
                    universal_newlines=True, # to improve the output rendering
                     )
    return process

# Executes a command on a worker and checks that it worked
def execute_command(worker, command, hostname=None):
    # we need a long timeout for the scp of the shuffles
    if command == 'scp':
        timeout = None
    else:
    # we can have a shorter timeout for mkdir commands
        timeout = 15
    try:
        # execute the sub process on a remote machine with a certain timeout
        process = launch_subprocess(worker, command, hostname)
        # try to get the process' outputs
        stdout, stderr = process.communicate(timeout=timeout)
        returncode = process.returncode

        # executed with errors
        if(returncode != 0):
            print('{}: \033[93m{} terminated with code {}\033[0m'.format(worker, command, returncode))
            if(len(stderr) > 0):
                print(' └ Errors: {}'.format(stderr.strip()))
            if(len(stdout) > 0):
                print(' └ Output: {}'.format(stdout.strip()))

        return returncode
    # the command timed out
    except subprocess.TimeoutExpired:
        print('{}: \033[91m{} timed out\033[0m'.format(worker, command))
        process.kill()
        return -1
    # we have another unexpected exception
    except Exception as e:
        print('{}: {} \033[91mthrew an exception {}\033[0m'.format(worker, command, e))
        process.kill()
        return -1

# the words maps function
def get_maps(filename):
    try:
        with open(filename, 'r', encoding='utf-8') as f:
            words_map = []

            # we read the file by line
            for line in f:
                 # we split the line by whitespace
                words = line.split()

                # we map the words list to the correct format
                # and we add it to our words_map list
                words_map.extend(map(lambda w: '{} 1\n'.format(w), words))

    except Exception:
        return None

    return words_map

# the words shuffle function
def get_shuffles(filename, workers, nb_workers, curdir, hostname):
    shuffles = {}
    try:
        # we read the content of the file created during the map
        with open(filename, 'r') as f:
            for line in f:
                # we generate the hash from each lines
                hash = generate_hash(line.split(' ')[0])
                #hashes.append(hash)

                # we perform attribution per machine using the hash
                modulo = int(hash) % nb_workers

                # first version
                # we write the content in a file in shuffles/<worker>/<hash>-<hostname>.txt
                # with open(curdir+'/shuffles/'+workers[modulo]+'/'+hash+'-'+hostname+'.txt', 'a') as f:
                #     f.writelines(line)

                # second version: we write one file per worker
                # we write the content in a file in shuffles/<worker>-<hostname>.txt
                # with open(curdir+'/shuffles/'+workers[modulo]+'-'+hostname+'.txt', 'a') as f:
                #     f.writelines(line)

                # final version: we write everything in RAM first
                if modulo in shuffles:
                    shuffles[modulo].append(line)
                else:
                    shuffles[modulo] = [line]

    except Exception:
        return False

    # final version: we copy the results from RAM to a file per worker in one shot
    for modulo in shuffles:
        # we write the shuffles in a file in shuffles/<worker>-<hostname>.txt
        try:
            with open(curdir+'/shuffles/'+workers[modulo]+'-'+hostname+'.txt', 'a') as f:
                f.writelines(shuffles[modulo])
        except Exception:
            return False
    return True

# Tries to execute commands on workers
def send_shuffle(worker_hostname_curdir):
    worker, (hostname, curdir) = worker_hostname_curdir
    # we check if there is a file for this worker, if not, we do nothing
    if(not os.path.exists(curdir+'/shuffles/'+worker+'-'+hostname+'.txt')):
        return True

    # we execute this 100*MAXRETRY times at most,
    # MAXRETRY times in case of scp error code other than 1
    retry_nb = 0
    for retry_nb in range(MAXRETRY*100):
        mkdir_returncode = execute_command(worker, 'mkdir')
        if mkdir_returncode == 0:
            scp_s_returncode = execute_command(worker, 'scp', hostname)
            if scp_s_returncode == 0:
                print('{}: \033[92mjob done\033[0m'.format(worker))
                #the job is successful
                return True
            # we had an unexpected error code
            if scp_s_returncode != 1 and retry_nb > MAXRETRY:
                break

        # we wait 200ms
        time.sleep(0.2)

    # the job failed
    return False

# the words reduce function
def get_reduces(filename, words_count):
    try:
        # we open the file
        with open(filename, encoding='utf-8') as f:
            # we read each line
            for line in f:
                # we get the word
                word = line.split(' ')[0]
                # we update our dict accordingly
                words_count[word] = words_count.get(word, 0) + 1

    except Exception:
        return False

    # we return our dict
    return True

# a function that generates a unique hash
def generate_hash(s):
    return str(int.from_bytes(hashlib.sha256(str.encode(s)).digest()[:4], 'little'))

def make_dir(dirname):
    try:
        os.makedirs(dirname, exist_ok=True)
    except FileExistsError:
        pass
    except OSError:
        # if we have an error while creating we exit with an error
        print('Not able to create directory maps', file=sys.stderr)
        return False

    return True

def read_workers(filename):
    workers = []
    with open(filename) as f:
        # we read the file by line
        for line in f:
            # we don't read lines that start with a # to allow for comments
            if line.startswith('#'):
                continue
            # we add the worker to the list of workers
            workers.append(line.strip())
    return workers

##
# main function
##
def main():
    # we check that we received the expected number of arguments
    if len(sys.argv) < 2:
        # we exit with an error if we didn't get the expected number of arguments
        print('Expected at least 1 argument, received none', file=sys.stderr)
        sys.exit(1)

    # we parse the option passed in argument
    option = sys.argv[1]

    if option in ['0', '1']:
        if len(sys.argv) < 3:
            # we exit with an error if we didn't get the expected number of arguments
            print('Expected at 2 arguments, received {}'.format(len(sys.argv)), file=sys.stderr)
            sys.exit(1)
        else: # we get the filename passed in arguement
            file = sys.argv[2]

    # we get our current working directory
    curdir = os.path.dirname(sys.argv[0])

    # if the current directory is empty, we replace it by '.' to avoid issues later
    if curdir == '':
        curdir = '.'

    ###
    # 0: get the splits and map
    if option == '0':
        # we extract the file number from the filename
        file_nb = re.search('([0-9]+)\.txt', file, re.IGNORECASE).group(1)

        # we perform the words map
        words_map = get_maps(file)

        # we couldn't open the file
        if words_map == None:
            print('Couldn\'t open file: {}'.format(file), file=sys.stderr)
            sys.exit(1)

        # we create the maps directory
        if not make_dir(curdir+'/maps'):
            # if we have an error while creating we exit with an error
            sys.exit(1)

        # we write the content in a file UM#id.txt
        with open(curdir+'/maps/UM'+file_nb+'.txt', 'w') as f:
            f.writelines(words_map)
    ###
    # option 1: shuffle
    elif option == '1':
        try:
            hostname = os.uname()[1]
        except Exception:
            hostname = DEFAULTHOST

        workers = read_workers(curdir+'/'+WORKERS_FILE)
        nb_workers = len(workers)

        # we create the shuffles directory
        if not make_dir(curdir+'/shuffles'):
            # if we have an error while creating we exit with an error
            sys.exit(1)

        # to create a directory per worker necessary for the previous versions
        # for worker in workers:
        #     if not make_dir(curdir+'/shuffles/'+worker):
        #         # if we have an error while creating we exit with an error
        #         sys.exit(1)

        shuffle_done = get_shuffles(file, workers, nb_workers, curdir, hostname)
        # we couldn't open the file
        if not shuffle_done:
            print('Couldn\'t open file: {}'.format(file), file=sys.stderr)
            sys.exit(2)

        # we generate a list of tuples (worker, filename) that attributes a worker to each filename
        pool = multiprocessing.Pool(nb_workers)
        # we determine which worker we need to send the file to
        workers_status = pool.map(send_shuffle, zip(workers, [(hostname,curdir)]*nb_workers))
        # if we have some failures
        if not all(workers_status):
            # we generate a list of the failed workers
            failed_workers = [w for w, s in zip(workers, workers_status) if not s]

            # we print a message detailing which machines failed
            print('Failed shuffle transfer to {}'.format(failed_workers), file=sys.stderr)
            sys.exit(3)

    ###
    # option 2: reduce
    elif option == '2':
        try:
            hostname = os.uname()[1]
        except Exception:
            hostname = DEFAULTHOST
        # we create the reduces directory
        if not make_dir(curdir+'/reduces'):
            # if we have an error while creating we exit with an error
            sys.exit(1)

        # we loop on each file received in shufflesreceived/
        words_count = {}
        for f in os.listdir(curdir+'/shufflesreceived/'):
            file = curdir+'/shufflesreceived/'+f
            #hash = f.split('-')[0]

            reduce_done = get_reduces(file, words_count)
            # we couldn't open the file
            if not reduce_done:
                print('Couldn\'t open file: {}'.format(file), file=sys.stderr)
                sys.exit(1)

        # Initial code to create one hash.txt per word
        # for (w,h), c in words_count.items():
        #     # we write the content in a file <hash>.txt in reduces/
        #     with open(curdir+'/reduces/'+h+'.txt', 'w') as f:
        #         f.writelines('{} {}\n'.format(w, c))

        # first improvement : we write line by line
        # we write our words count into a file in /reduces
        # for w, c in words_count.items():
        #     with open(curdir+'/reduces/'+hostname+'.txt', 'a') as f:
        #         f.writelines('{} {}\n'.format(w, c))

        # final version : we write the reduce file in one shot
        with open(curdir+'/reduces/'+hostname+'.txt', 'w') as f:
            f.writelines(map(lambda wc: '{} {}\n'.format(wc[0], wc[1]), words_count.items()))

if __name__ == '__main__':
    main()


from typing import List

class Solution:
    def rob(self, nums: List[int]) -> int:
        if(len(nums)==0):
            return 0
        if(len(nums)==1):
            return nums[0]
        dp=[0]*len(nums)
        dp[0]=nums[0]
        dp[1]=max(nums[0],nums[1])
        for i in range(2,len(nums)):
            dp[i]=max(nums[i]+dp[i-2],dp[i-1])
        return dp[len(nums)-1]

main=Solution()
a= [2,1]
print(main.rob(a))


import cv2

# Scaling
# https://docs.opencv.org/4.2.0/da/d6e/tutorial_py_geometric_transformations.html

# Read an image using cv2.imread
img = cv2.imread('../../resources/lena.png')

# The size of the image can be specified manually, or you can specify the scaling factor
# Preferable interpolation methods are cv2.INTER_AREA for shrinking and cv2.INTER_CUBIC (slow)
# and cv2.INTER_LINEAR for zooming
img_scaled = cv2.resize(img, None, fx=2, fy=2, interpolation=cv2.INTER_CUBIC)
cv2.imshow('Bigger Image', img_scaled)

img_scaled = cv2.resize(img, None, fx=0.5, fy=0.5, interpolation=cv2.INTER_CUBIC)
cv2.imshow('Smaller Image', img_scaled)

# Rezise modifing the number of pixels
img_scaled = cv2.resize(img, (1000, 500))
cv2.imshow('Modify pixels Image 1', img_scaled)

img_scaled = cv2.resize(img, (300, 1000))
cv2.imshow('Modify pixels Image 2', img_scaled)

cv2.waitKey(0)
cv2.destroyAllWindows()


import requests

if __name__ == "__main__":

    #添加文章
    data = {'title':'test123','content':'101010'}
    r = requests.post("http://127.0.0.1:8000/article/",data = data)
    print(r.text)

    # # 修改文章
    # data = {'id':1,'content':'修改文章内容'}
    # r = requests.put("http://127.0.0.1:8000/article/",data = data)
    # print(r.text)

    # 删除文章
    # r = requests.delete("http://127.0.0.1:8000/article/?id=2")
    # print(r.text)


import pika
from time import sleep
#用户名和密码
credentials = pika.PlainCredentials('kudoxi','kudoxi')
#连接机器
connection = pika.BlockingConnection(pika.ConnectionParameters(
    '192.168.254.138',credentials=credentials
))

#建立rabbitmq协议通道
channel = connection.channel()

#声明队列，从哪里接收
channel.queue_declare(queue='task_queue',durable=True)
#send里面已经设置为了持久化，这里声明队列也必须设置为持久化，队列一旦声明就不能再更改

#ch 通道channel的实例
def callback(ch,method,properties,body):
    print("start processing ... %r"%body)
    sleep(20)
    print("[x] done ")
    print("[x] message.delivery_tag ",method.delivery_tag)
    ch.basic_ack(delivery_tag=method.delivery_tag)#发送一个随机值，作为响应标记符

#从队列拿到数据后调取callback---注意，不同版本，参数位置不一样
channel.basic_consume("task_queue",callback,auto_ack=False)
#auto_ack 是否给队列发送标识符标识自己已经接收，如果是TRUE，那么会自动发送，即使中断了，队列中的消息也被拿走了
#如果是false，中断之后，队列中的消息还在，一定要完整执行结束，消息才会被释放

#开始消费
print('[*]waiting for messages.To exit press ctrl+c')
channel.start_consuming()


import telebot
from telebot import types

api = 'fig tebe token'

marking = 'Маркировка - нанесение на упаковку или этикетку сгенерированного цифрового кода: электронного точечного двухмерного кода формата Data Matrix. Основной целью маркировки товаров является повышение защиты и снижение оборота фальсифицированной и контрафактной продукции.'
kkm = 'ККМ (Контрольно-кассовая машина) - аппаратно-программные средства с фискальной памятью или программное обеспечение с функцией фиксации, некорректируемой ежесуточной (ежесменной) регистрацией, долговременного хранения и передачи данных в режиме реального времени в уполномоченный налоговый орган в защищенном виде.'
ofd = 'ОФД (Оператор фискальных данных) – это организация, обеспечивающая технический процесс сбора и передачи данных в защищенном виде только уполномоченному налоговому органу или оператору национальной системы маркировки и прослеживаемости маркированных товаров в Кыргызской Республике.'

bot = telebot.TeleBot(api)

keyboard_1 = telebot.types.ReplyKeyboardMarkup(True)

btn_1 = telebot.types.KeyboardButton('Маркировка')
btn_2 = telebot.types.KeyboardButton('ККМ')
btn_3 = telebot.types.KeyboardButton('ОФД')
btn_4 = telebot.types.KeyboardButton('Автоматизация магазина')
btn_5 = telebot.types.KeyboardButton('Больше информации!')

keyboard_1.row(btn_1, btn_2)
keyboard_1.row(btn_3, btn_4)
keyboard_1.row(btn_5)

@bot.message_handler(commands = ['url'])
def url(message):
    markup = types.InlineKeyboardMarkup()
    btn_my_site = types.InlineKeyboardButton(text='Наш сайт', url='http://www.norma.kg/')
    markup.add(btn_my_site)
    bot.send_message(message.chat.id, "Нажми на кнопку и перейди на наш сайт.", reply_markup=markup)

@bot.message_handler(commands=['start'])
def start_message(message):
    bot.send_message(message.chat.id, 'Здравствуйте! Вас приветствует Группа Компаний «Новая Норма»! ', reply_markup=keyboard_1)

@bot.message_handler(content_types=['text'])
# def info_message(message):
#     bot.send_message(message.chat.id, , reply_markup=keyboard_1)
def send_text(message):
    if message.text.lower() == 'маркировка':
        bot.send_photo(chat_id=message.chat.id, caption=marking, photo='http://www.norma.kg/static/media/Rectangle%2063.40da64e4.png', reply_markup=keyboard_1)
    if message.text.lower() == 'ккм':
        bot.send_photo(chat_id=message.chat.id, caption=kkm, photo='https://smebanking.news/ru/wp-content/uploads/sites/2/2018/10/online-kassa.jpg', reply_markup=keyboard_1)
        # bot.send_photo(chat_id=message.chat.id, photo='http://www.norma.kg/static/media/pic1.b9546ea8.png')
    if message.text.lower() == 'офд':
            bot.send_photo(chat_id=message.chat.id, caption=ofd, photo='http://www.norma.kg/static/media/Rectangle%2016.bea67cda.png', reply_markup=keyboard_1)
            # bot.send_photo(chat_id=message.chat.id, photo='http://www.norma.kg/static/media/Rectangle%2016.bea67cda.png')
    if message.text.lower() == 'автоматизация магазина':
                bot.send_photo(chat_id=message.chat.id, caption='Установка товаро-учетных систем:\n- 1С;\n- Мой Склад и т.д.',
                      photo='https://brigit.com.ua/wp-content/uploads/retail.jpg', reply_markup=keyboard_1)
    #             bot.send_photo(chat_id=message.chat.id, photo='http://www.norma.kg/static/media/pic1.b9546ea8.png')
    if message.text == 'Больше информации!':
            bot.send_message(chat_id=message.chat.id, text='Номер нашего call-центра: +996501588882\n'
                    'Наш сайт: http://www.norma.kg/\n'
                    'Эл. почта: nnormakg@gmail.com\n'
                    'inst : https://www.instagram.com/norma.kg/?hl=ru ', reply_markup=keyboard_1)

@bot.message_handler(content_types=['text'])
def go_txt(message):
    if message.text == 'Привет':
        bot.send_message(message.chat.id, 'Здраствуйте!')
    elif message.text == 'Пока':
        bot.send_message(message.chat.id, 'Всего доброго!')
    elif message.text == 'ха-ха':
        bot.send_sticker(message.chat.id, 'CAADAgADcwgAAhhC7ggBnQGJ6b93ggI')
    else:
        bot.send_message(message.chat.id, 'Я Вас не понимаю ')

bot.polling()


# -*- coding: utf-8 -*-
"""
Created on Sun Jan  1 18:05:08 2023

@author: Rahul
"""
import warnings
warnings.filterwarnings('ignore')

import pandas as pd
import numpy as np

df = pd.read_csv("D:\\DS\\books\\ASSIGNMENTS\\Naive Bayes\\SalaryData_Test.csv")
df
df.info()
df.shape

df.head()
df.dtypes

#Finding the special characters in the data frame
df.isin(['?']).sum(axis=0)
print(df[0:5])

df.native.value_counts()
df.native.unique()

df.workclass.value_counts()
df.workclass.unique()

df.occupation.value_counts()
df.occupation.unique()

df.sex.value_counts()

#finding categorical and numerical
cat = [var for var in df.columns if df[var].dtype=='O']
print('There are {} categorical variables\n'.format(len(cat)))
print('The categorical variables are :\n\n', cat)

#find numerical variables
num = [var for var in df.columns if df[var].dtype!='O']
print('There are {} numerical variables\n'.format(len(num)))
print('The numerical variables are :', num)

# check if there are any missing values missing values in categorical variables

df[cat].isnull().sum()

df[num].isnull().sum()

# Visualisation
import seaborn as sns
import matplotlib.pyplot as plt

df["age"].hist()
df["educationno"].hist()
df["capitalgain"].hist()
df["capitalloss"].hist()
df["hoursperweek"].hist()

t1 = pd.crosstab(index=df["education"],columns=df["workclass"])
t1.plot(kind='bar')

t2 = pd.crosstab(index=df["education"],columns=df["Salary"])
t2.plot(kind='bar')

t3 = pd.crosstab(index=df["sex"],columns=df["race"])
t3.plot(kind='bar')

t4 = pd.crosstab(index=df["maritalstatus"],columns=df["sex"])
t4.plot(kind='bar')

# Outliers dectection and treating outliers
df.boxplot("age",vert=False)
Q1=np.percentile(df["age"],25)
Q3=np.percentile(df["age"],75)
IQR=Q3-Q1
LW=Q1-(2.0*IQR)
UW=Q3+(2.0*IQR)
df["age"]<LW
df[df["age"]<LW]
df[df["age"]<LW].shape
df["age"]>UW
df[df["age"]>UW]
df[df["age"]>UW].shape
df["age"]=np.where(df["age"]>UW,UW,np.where(df["age"]<LW,LW,df["age"]))

df.boxplot("educationno",vert=False)
Q1=np.percentile(df["educationno"],25)
Q3=np.percentile(df["educationno"],75)
IQR=Q3-Q1
LW=Q1-(2.0*IQR)
UW=Q3+(2.0*IQR)
df["educationno"]<LW
df[df["educationno"]<LW]
df[df["educationno"]<LW].shape
df["educationno"]>UW
df[df["educationno"]>UW]
df[df["educationno"]>UW].shape
df["educationno"]=np.where(df["educationno"]>UW,UW,np.where(df["educationno"]<LW,LW,df["educationno"]))

df.boxplot("hoursperweek",vert=False)
Q1=np.percentile(df["hoursperweek"],25)
Q3=np.percentile(df["hoursperweek"],75)
IQR=Q3-Q1
LW=Q1-(2.0*IQR)
UW=Q3+(2.0*IQR)
df["hoursperweek"]<LW
df[df["hoursperweek"]<LW]
df[df["hoursperweek"]<LW].shape
df["hoursperweek"]>UW
df[df["hoursperweek"]>UW]
df[df["hoursperweek"]>UW].shape
df["hoursperweek"]=np.where(df["hoursperweek"]>UW,UW,np.where(df["hoursperweek"]<LW,LW,df["hoursperweek"]))

df.boxplot("capitalgain",vert=False)
Q1=np.percentile(df["capitalgain"],25)
Q3=np.percentile(df["capitalgain"],75)
IQR=Q3-Q1
LW=Q1-(2.0*IQR)
UW=Q3+(2.0*IQR)
df["capitalgain"]<LW
df[df["capitalgain"]<LW]
df[df["capitalgain"]<LW].shape
df["capitalgain"]>UW
df[df["capitalgain"]>UW]
df[df["capitalgain"]>UW].shape
df["capitalgain"]=np.where(df["capitalgain"]>UW,UW,np.where(df["capitalgain"]<LW,LW,df["capitalgain"]))

df.boxplot("capitalloss",vert=False)
Q1=np.percentile(df["capitalloss"],25)
Q3=np.percentile(df["capitalloss"],75)
IQR=Q3-Q1
LW=Q1-(2.0*IQR)
UW=Q3+(2.0*IQR)
df["capitalloss"]<LW
df[df["capitalloss"]<LW]
df[df["capitalloss"]<LW].shape
df["capitalloss"]>UW
df[df["capitalloss"]>UW]
df[df["capitalloss"]>UW].shape
df["capitalloss"]=np.where(df["capitalloss"]>UW,UW,np.where(df["capitalloss"]<LW,LW,df["capitalloss"]))

# Spliting
X=df.iloc[:,0:13]
X.columns
Y=df["Salary"]

# split X and y into training and testing sets

from sklearn.model_selection import train_test_split
X_train, X_test, y_train, y_test = train_test_split(X, Y, test_size = 0.3, random_state = 0)

# display categorical variables
categorical = [col for col in X_train.columns if X_train[col].dtypes == 'O']
categorical

# display numerical variables
numerical = [col for col in X_train.columns if X_train[col].dtypes != 'O']
numerical

#Data Transformation
#pip install category_encoders
import category_encoders as ce

encoder = ce.OneHotEncoder(cols=['workclass', 'education', 'maritalstatus', 'occupation', 'relationship',
                                 'race', 'sex', 'native'])
X_train = encoder.fit_transform(X_train)
X_test = encoder.transform(X_test)
X_train.head()

from sklearn.preprocessing import MinMaxScaler #fixed import

scaler = MinMaxScaler()
X_train = scaler.fit_transform(X_train)
X_test = scaler.transform(X_test)

#model fitting
from sklearn.naive_bayes import GaussianNB
gnb = GaussianNB()
# fit the model
gnb.fit(X_train, y_train)

y_pred_train = gnb.predict(X_train)
y_pred_test = gnb.predict(X_test)

from sklearn.metrics import accuracy_score
print("trainin score:",accuracy_score(y_train, y_pred_train).round(2))
print("test score:",accuracy_score(y_test, y_pred_test).round(2))

from sklearn.naive_bayes import BernoulliNB
bn = BernoulliNB()

bn.fit(X_train, y_train)

y_pred_train = bn.predict(X_train)
y_pred_test = bn.predict(X_test)

from sklearn.metrics import accuracy_score
print("trainin score:",accuracy_score(y_train, y_pred_train).round(2))
print("test score:",accuracy_score(y_test, y_pred_test).round(2))

from sklearn.naive_bayes import MultinomialNB
mb = MultinomialNB()

mb.fit(X_train, y_train)

y_pred_train = mb.predict(X_train)
y_pred_test = mb.predict(X_test)

from sklearn.metrics import accuracy_score
print("trainin score:",accuracy_score(y_train, y_pred_train).round(2))
print("test score:",accuracy_score(y_test, y_pred_test).round(2))

from sklearn.svm import SVC

svc_class = SVC(kernel='linear')

svc_class.fit(X_train,y_train)

y_predict_train = svc_class.predict(X_train)
y_predict_test = svc_class.predict(X_test)

print("trainin score:",accuracy_score(y_train, y_pred_train).round(2))
print("test score:",accuracy_score(y_test, y_pred_test).round(2))

from sklearn.linear_model import LogisticRegression

logreg = LogisticRegression()

logreg.fit(X_train,y_train)

#prediction
y_pred_train = logreg.predict(X_train)
y_pred_test = logreg.predict(X_test)

print("trainin score:",accuracy_score(y_train, y_pred_train).round(2))
print("test score:",accuracy_score(y_test, y_pred_test).round(2))