import mxnet.ndarray as nd
from mxnet.contrib.autograd import *
from mxnet.test_utils import *

def autograd_assert(*args, **kwargs):
    func   = kwargs["func"]
    grad_f = kwargs["grad_func"]
    argnum = kwargs["argnum"] if 'argnum' in kwargs else None

    grad_func = grad_and_loss(func, argnum)
    grad_vals, output = grad_func(*args)
    res = func(*args)
    assert same(output.asnumpy(), res.asnumpy())
    grad_res = grad_f(*args)
    assert len(grad_vals) == len(grad_res)
    for a, b in zip(grad_vals, grad_res):
        assert same(a.asnumpy(), b.asnumpy())

def test_unary_func():
    x = nd.uniform(shape=(4, 5))
    f_exp         = lambda x: nd.exp(x)
    f_exp_grad    = lambda x: [nd.exp(x)]
    autograd_assert(x, func=f_exp, grad_func=f_exp_grad)
    f_half        = lambda x: x/2
    f_half_grad   = lambda x: [nd.ones(x.shape) * 0.5]
    autograd_assert(x, func=f_half, grad_func=f_half_grad)
    f_square      = lambda x: x**2
    f_square_grad = lambda x: [2*x]
    autograd_assert(x, func=f_square, grad_func=f_square_grad)

def test_binary_func():
    x = nd.uniform(shape=(4, 5))
    y = nd.uniform(shape=(4, 5))
    f_add      = lambda x, y: x+y
    f_add_grad = lambda x, y: [nd.ones(x.shape), nd.ones(y.shape)]
    autograd_assert(x, y, func=f_add, grad_func=f_add_grad)
    f_mul      = lambda x, y: x*y
    f_mul_grad = lambda x, y: [y, x]
    autograd_assert(x, y, func=f_mul, grad_func=f_mul_grad)
    f_compose  = lambda x, y: x+x*y
    f_compose_grad = lambda x, y: [nd.ones(x.shape) + y, x]
    autograd_assert(x, y, func=f_compose, grad_func=f_compose_grad)

def test_operator_with_state():
    def f_fc(a, b, weight, bias):
        x = a*b
        fc = nd.FullyConnected(
            x, weight, bias, num_hidden=32)
        return fc

    a = nd.uniform(shape=(64, 50))
    b = nd.uniform(shape=(64, 50))
    weight = nd.uniform(shape=(32, 50))
    bias = nd.uniform(shape=(32, ))

    grad_func = grad_and_loss(f_fc)
    grad_vals, outputs = grad_func(a, b, weight, bias)
    # (TODO) assert

def test_argnum():
    def f_with_mode(a, b, mode):
        if mode:
            return a+b
        else:
            return a*b

    a = nd.uniform(shape=(3, 2))
    b = nd.uniform(shape=(3, 2))
    f_add_grad = lambda x, y, mode: [nd.ones(x.shape), nd.ones(y.shape)]
    f_mul_grad = lambda x, y, mode: [y, x]
    autograd_assert(a, b, True,
        argnum=[0, 1], func=f_with_mode, grad_func=f_add_grad)
    autograd_assert(a, b, False,
        argnum=[0, 1], func=f_with_mode, grad_func=f_mul_grad)

def test_training():
    x = nd.ones((10, 10))
    with train_section():
        y = nd.Dropout(x, p=0.5)
        assert not (y.asnumpy() == x.asnumpy()).all()
        with test_section():
            y = nd.Dropout(x, p=0.5)
            assert (y.asnumpy() == x.asnumpy()).all()

def test_out_grads():
    x = nd.ones((3, 5))
    dx = nd.zeros_like(x)
    mark_variables([x], [dx])
    da = None
    db = nd.array([1,2,3,4,5])
    dc = nd.array([5,4,3,2,1])

    with train_section():
        a, b, c = nd.split(x, axis=0, num_outputs=3, squeeze_axis=True)
        backward([a, b, c], [da, db, dc])

    assert (dx.asnumpy() == np.array(
        [[1,1,1,1,1],
         [1,2,3,4,5],
         [5,4,3,2,1]])).all()

def test_detach_updated_grad():
    x = nd.ones((2, 2))
    dx = nd.zeros_like(x)
    y = nd.ones_like(x)
    dy = nd.zeros_like(x)
    mark_variables([x, y], [dx, dy])
    assert x._fresh_grad == False
    assert y._fresh_grad == False

    with train_section():
        x2 = x + 2
        y2  = x2 + y
        y2.backward()
    assert (dx.asnumpy() == 1).all()
    assert x._fresh_grad == True
    assert y._fresh_grad == True

    dx[:] = 0
    x._fresh_grad = False
    y._fresh_grad = False
    assert x._fresh_grad == False
    assert y._fresh_grad == False
    with train_section():
        x2 = x + 2
        x2 = x2.detach()
        y2  = x2 + y
        y2.backward()
    assert (dx.asnumpy() == 0).all()
    assert y._fresh_grad == True
    assert x._fresh_grad == False

def test_retain_grad():
    x = mx.nd.ones((2, 2))
    dx = mx.nd.zeros((2, 2))
    mark_variables([x], [dx], grad_reqs='add')
    with train_section():
        y = x + 1
        y.backward(retain_graph=False)
    assert (dx.asnumpy() == 1).all()

    dx[:] = 0
    with train_section():
        y = x + 1
        y.backward(retain_graph=True)
        y.backward(retain_graph=False)
    assert (dx.asnumpy() == 2).all()

    try:
        with train_section():
            y = x + 1
            y.backward()
            y.backward()
    except Exception:
        return

    raise AssertionError(
        "differentiating the same graph twice without retain_graph should fail")

if __name__ == "__main__":
    import nose
    nose.runmodule()


# -*- coding: utf-8 -*-
# @Time    : 2021/6/9 11:12
# @File    : 265. 粉刷房子 II.py
from leetcode import *

# 理解错题意，题目说是相邻房子不同，不是连续两次不同
class Solution:
    def minCostII(self, costs: List[List[int]]) -> int:
        # 0 连续涂两次 1 没有连续涂两次
        dp = [[0] * 2 for _ in costs]
        p, q = 0, 0  # 记录涂的下标 p 连续两次下标 q 没有连续两次下标
        q = costs[0].index(min(costs[0]))
        dp[0][1] = costs[0][q]
        for i in range(1, len(costs)):
            dp[i][0] = dp[i - 1][1] + costs[i][p]
            if costs[i][:q] and costs[i][q + 1:]:
                tmp_q = costs[i].index(min(min(costs[i][:q]), min(costs[i][q + 1:])))
            elif not costs[i][:q]:
                tmp_q = costs[i].index(min(costs[i][q + 1:]))
            else:
                tmp_q = costs[i].index(min(costs[i][:q]))
            if costs[i][:p] and costs[i][p + 1:]:
                tmp_p = costs[i].index(min(min(costs[i][:p]), min(costs[i][p + 1:])))
            elif not costs[i][:p]:
                tmp_p = costs[i].index(min(costs[i][p + 1:]))
            else:
                tmp_p = costs[i].index(min(costs[i][:p]))
            if dp[i - 1][1] + costs[i][p] < dp[i - 1][0] + costs[i][q]:
                dp[i][1] = dp[i - 1][1] + costs[i][p]
                q = tmp_q
            else:
                dp[i][1] = dp[i - 1][1] + costs[i][p]
                p = tmp_p
        return min(dp[-1])

# dp[i][j] 表示第i个房间采取第j个涂料时的最小代价
class Solution:
    def minCostII(self, costs: List[List[int]]) -> int:
        def findMinIndex(nums, dismiss):  # 找到nums中下标不为dismiss的最小数的下标
            curIndex = 0 if dismiss != 0 else 1
            for i in range(len(nums)):
                if nums[i] < nums[curIndex] and i != dismiss:
                    curIndex = i
            return curIndex

        dp = [[0] * len(costs[0]) for _ in costs]
        for j in range(len(costs[0])):
            dp[0][j] = costs[0][j]
        for i in range(1, len(costs)):
            for j in range(len(costs[0])):
                index = findMinIndex(dp[i - 1], j)
                dp[i][j] = dp[i - 1][index] + costs[i][j]
        return min(dp[-1])

# 优化 用min_id记录前一次选取的下标，如果这一次与前一次下标不同再查找
class Solution:
    def minCostII(self, costs: List[List[int]]) -> int:
        n = len(costs)
        if n == 0:
            return 0
        k = len(costs[0])

        dp = [[0 for i in range(k)] for i in range(n)]

        for i in range(n):
            if i == 0:
                dp[i] = costs[i]
                # min_id = dp[0].index(min(dp[0]))
            else:
                for j in range(k):
                    if j != min_id:
                        dp[i][j] = costs[i][j] + dp[i - 1][min_id]
                    else:
                        dp[i][j] = costs[i][j] + min(dp[i - 1][0:j] + dp[i - 1][j + 1:])
            min_id = dp[i].index(min(dp[i]))

        return min(dp[n - 1])


import requests

eve = "http://eve.gameloft.com"

#2148:58919:3.9.0g:android:googleplay

clientID = raw_input('Enter your ClientID:  ')
urlEve = eve +  "/config/" + clientID + "/datacenters"

# sending get request
def readURL(URL):
	return requests.get(URL)

# extracting data in json format
configEve = readURL(urlEve).json()
for nameDatacenters in configEve:
	urlPandora = urlEve + "/"+ nameDatacenters['name'] + "/urls"
	dataPandora = readURL(urlPandora).json()
	print(nameDatacenters['name'].upper() + "-" + dataPandora['pandora'])


import subprocess

def execute_process(command: str, to_null: bool):
    """Executes an external shell process.

    Args:
        command: The string of the command to execute.
        to_null: Determines where to send output.
    Raises:
        Exception: If an error occurs in executing the script.
    """
    stdout = subprocess.DEVNULL if to_null else None
    try:
        subprocess.run([command], shell=True, check=True,
            stdout=stdout,
            stderr=subprocess.STDOUT)
    except subprocess.CalledProcessError as err:
        raise RuntimeError(f'Error executing process. {err}') from err

# Install AutoMLOps from [PyPI](https://pypi.org/project/google-cloud-automlops/), or locally by cloning the repo and running `pip install .`
execute_process('pip3 install google-cloud-automlops --user', False)

# Set your project ID below.
PROJECT_ID = 'automlops-sandbox'  # @param {type:"string"}

# Set your Model ID below.
MODEL_ID = 'dry-beans-dt'

# # 1. AutoMLOps Pipeline
# This workflow will define and generate a pipeline using AutoMLOps. AutoMLOps provides 2 functions for defining MLOps pipelines:

# - `AutoMLOps.component(...)`: Defines a component, which is a containerized python function.
# - `AutoMLOps.pipeline(...)`: Defines a pipeline, which is a series of components.

# AutoMLOps provides 5 functions for building and maintaining MLOps pipelines:

# - `AutoMLOps.generate(...)`: Generates the MLOps codebase. Users can specify the tooling and technologies they would like to use in their MLOps pipeline.
# - `AutoMLOps.provision(...)`: Runs provisioning scripts to create and maintain necessary infra for MLOps.
# - `AutoMLOps.deprovision(...)`: Runs deprovisioning scripts to tear down MLOps infra created using AutoMLOps.
# - `AutoMLOps.deploy(...)`: Builds and pushes component container, then triggers the pipeline job.
# - `AutoMLOps.launchAll(...)`: Runs `generate()`, `provision()`, and `deploy()` all in succession.

# Please see the [readme](https://github.com/GoogleCloudPlatform/automlops/blob/main/README.md) for more information.

# Import AutoMLOps
from google_cloud_automlops import AutoMLOps

# ## Data Loading
# Define a custom component for loading and creating a dataset using `@AutoMLOps.component`. Import statements and helper functions must be added inside the function. Provide parameter type hints.
#
# **Note: we currently only support python primitive types for component parameters. If you would like to use something more advanced, please use the Kubeflow spec instead (see below in this notebook).**

@AutoMLOps.component(
    packages_to_install=[
        'google-cloud-bigquery',
        'pandas',
        'pyarrow',
        'db_dtypes',
        'fsspec',
        'gcsfs'
    ]
)
def create_dataset(
    bq_table: str,
    data_path: str,
    project_id: str
):
    """Custom component that takes in a BQ table and writes it to GCS.

    Args:
        bq_table: The source biquery table.
        data_path: The gcs location to write the csv.
        project_id: The project ID.
    """
    from google.cloud import bigquery
    import pandas as pd
    from sklearn import preprocessing

    bq_client = bigquery.Client(project=project_id)

    def get_query(bq_input_table: str) -> str:
        """Generates BQ Query to read data.

        Args:
        bq_input_table: The full name of the bq input table to be read into
        the dataframe (e.g. <project>.<dataset>.<table>)
        Returns: A BQ query string.
        """
        return f'''
        SELECT *
        FROM `{bq_input_table}`
        '''

    def load_bq_data(query: str, client: bigquery.Client) -> pd.DataFrame:
        """Loads data from bq into a Pandas Dataframe for EDA.
        Args:
        query: BQ Query to generate data.
        client: BQ Client used to execute query.
        Returns:
        pd.DataFrame: A dataframe with the requested data.
        """
        df = client.query(query).to_dataframe()
        return df

    dataframe = load_bq_data(get_query(bq_table), bq_client)
    le = preprocessing.LabelEncoder()
    dataframe['Class'] = le.fit_transform(dataframe['Class'])
    dataframe.to_csv(data_path)

# ## Model Training
# Define a custom component for training a model using `@AutoMLOps.component`. Import statements and helper functions must be added inside the function.

@AutoMLOps.component(
    packages_to_install=[
        'scikit-learn==1.2.0',
        'pandas',
        'joblib',
        'tensorflow'
    ]
)
def train_model(
    data_path: str,
    model_directory: str
):
    """Custom component that trains a decision tree on the training data.

    Args:
        data_path: GS location where the training data.
        model_directory: GS location of saved model.
    """
    from sklearn.tree import DecisionTreeClassifier
    from sklearn.model_selection import train_test_split
    import pandas as pd
    import tensorflow as tf
    import pickle
    import os

    def save_model(model, uri):
        """Saves a model to uri."""
        with tf.io.gfile.GFile(uri, 'w') as f:
            pickle.dump(model, f)

    df = pd.read_csv(data_path)
    labels = df.pop('Class').tolist()
    data = df.values.tolist()
    x_train, x_test, y_train, y_test = train_test_split(data, labels)
    skmodel = DecisionTreeClassifier()
    skmodel.fit(x_train,y_train)
    score = skmodel.score(x_test,y_test)
    print('accuracy is:',score)

    output_uri = os.path.join(model_directory, 'model.pkl')
    save_model(skmodel, output_uri)

# ## Uploading & Deploying the Model
# Define a custom component for uploading and deploying a model in Vertex AI, using `@AutoMLOps.component`. Import statements and helper functions must be added inside the function.

@AutoMLOps.component(
    packages_to_install=[
        'google-cloud-aiplatform'
    ]
)
def deploy_model(
    model_directory: str,
    project_id: str,
    region: str
):
    """Custom component that trains a decision tree on the training data.

    Args:
        model_directory: GS location of saved model.
        project_id: Project_id.
        region: Region.
    """
    from google.cloud import aiplatform

    aiplatform.init(project=project_id, location=region)
    # Check if model exists
    models = aiplatform.Model.list()
    model_name = 'beans-model'
    if 'beans-model' in (m.name for m in models):
        parent_model = model_name
        model_id = None
        is_default_version=False
        version_aliases=['experimental', 'challenger', 'custom-training', 'decision-tree']
        version_description='challenger version'
    else:
        parent_model = None
        model_id = model_name
        is_default_version=True
        version_aliases=['champion', 'custom-training', 'decision-tree']
        version_description='first version'

    serving_container = 'us-docker.pkg.dev/vertex-ai/prediction/sklearn-cpu.1-0:latest'
    uploaded_model = aiplatform.Model.upload(
        artifact_uri = model_directory,
        model_id=model_id,
        display_name=model_name,
        parent_model=parent_model,
        is_default_version=is_default_version,
        version_aliases=version_aliases,
        version_description=version_description,
        serving_container_image_uri=serving_container,
        serving_container_ports=[8080],
        labels={'created_by': 'automlops-team'},
    )

    endpoint = uploaded_model.deploy(
        machine_type='n1-standard-4',
        deployed_model_display_name='deployed-beans-model')

# ## Define the Pipeline
# Define your pipeline using `@AutoMLOps.pipeline`. You can optionally give the pipeline a name and description. Define the structure by listing the components to be called in your pipeline; use `.after` to specify the order of execution.

@AutoMLOps.pipeline #(name='automlops-pipeline', description='This is an optional description')
def pipeline(bq_table: str,
             model_directory: str,
             data_path: str,
             project_id: str,
             region: str,
            ):

    create_dataset_task = create_dataset(
        bq_table=bq_table,
        data_path=data_path,
        project_id=project_id)

    train_model_task = train_model(
        model_directory=model_directory,
        data_path=data_path).after(create_dataset_task)

    deploy_model_task = deploy_model(
        model_directory=model_directory,
        project_id=project_id,
        region=region).after(train_model_task)

# ## Define the Pipeline Arguments
import datetime
pipeline_params = {
    'bq_table': f'{PROJECT_ID}.test_dataset.dry-beans',
    'model_directory': f'gs://{PROJECT_ID}-bucket/trained_models/{datetime.datetime.now()}',
    'data_path': f'gs://{PROJECT_ID}-bucket/data',
    'project_id': f'{PROJECT_ID}',
    'region': 'us-central1'
}

# ## Generate and Run the pipeline
# `AutoMLOps.generate(...)` generates the MLOps codebase. Users can specify the tooling and technologies they would like to use in their MLOps pipeline.
AutoMLOps.generate(project_id=PROJECT_ID,
                   pipeline_params=pipeline_params,
                   use_ci=True,
                   naming_prefix=MODEL_ID,
                   schedule_pattern='59 11 * * 0' # retrain every Sunday at Midnight
)


from base64 import standard_b64encode
import sys

def serialize_gr_command(cmd, payload=None):
    cmd = ','.join('{}={}'.format(k, v) for k, v in cmd.items())
    ans = []
    w = ans.append
    w(b'\033_G'), w(cmd.encode('ascii'))
    if payload:
        w(b';')
        w(payload)
    w(b'\033\\')
    return b''.join(ans)

def write_chunked(cmd, data, b64=False):
    """
    write data in chunks to STDOUT.

    Args:
        cmd: command dict
        data: data to send
        b64: is data already b64 encoded?
    """

    if not b64:
        data = standard_b64encode(data)
    while data:
        chunk, data = data[:4096], data[4096:]
        m = 1 if data else 0
        cmd['m'] = m
        sys.stdout.buffer.write(serialize_gr_command(cmd, chunk))
        sys.stdout.flush()
        cmd.clear()