from __future__ import print_function
import argparse
import torch
import torch.nn.functional as F
from torchvision import datasets, transforms
from models import vgg, resnet, densenet
import numpy as np
import os
import sys
from tqdm import tqdm
from utils import *

if __name__ == '__main__':

	parser = argparse.ArgumentParser(description='Cifar10 Evaluation')
	parser.add_argument('--cp-path', type=str, default=None, metavar='Path', help='Path for checkpointing')
	parser.add_argument('--data-path', type=str, default='./data/', metavar='Path', help='Path to data')
	parser.add_argument('--batch-size', type=int, default=100, metavar='N', help='input batch size for testing (default: 100)')
	parser.add_argument('--model', choices=['vgg', 'resnet', 'densenet'], default='resnet')
	parser.add_argument('--dropout-prob', type=float, default=0.25, metavar='p', help='Dropout probability (default: 0.25)')
	parser.add_argument('--no-cuda', action='store_true', default=False, help='Disables GPU use')
	parser.add_argument('--workers', type=int, default=4, metavar='N', help='Data load workers (default: 4)')
	args = parser.parse_args()
	args.cuda = True if not args.no_cuda and torch.cuda.is_available() else False

	transform_test = transforms.Compose([transforms.ToTensor(), transforms.Normalize([x / 255 for x in [125.3, 123.0, 113.9]], [x / 255 for x in [63.0, 62.1, 66.7]])])
	testset = datasets.CIFAR10(root='./data', train=False, download=True, transform=transform_test)
	test_loader = torch.utils.data.DataLoader(testset, batch_size=args.batch_size, shuffle=False, num_workers=args.workers)

	ckpt = torch.load(args.cp_path, map_location = lambda storage, loc: storage)
	try :
		dropout_prob, n_hidden, hidden_size, softmax = ckpt['dropout_prob'], ckpt['n_hidden'], ckpt['hidden_size'], ckpt['sm_type']
	except KeyError as err:
		print("Key Error: {0}".format(err))
		print('\nProbably old cp has no info regarding classifiers arch!\n')
		n_hidden, hidden_size, softmax = get_classifier_config_from_cp(ckpt)
		dropout_prob = args.dropout_prob

	if args.model == 'vgg':
		model = vgg.VGG('VGG16', nh=n_hidden, n_h=hidden_size, dropout_prob=dropout_prob, sm_type=softmax)
	elif args.model == 'resnet':
		model = resnet.ResNet18(nh=n_hidden, n_h=hidden_size, dropout_prob=dropout_prob, sm_type=softmax)
	elif args.model == 'densenet':
		model = densenet.densenet_cifar(nh=n_hidden, n_h=hidden_size, dropout_prob=dropout_prob, sm_type=softmax)

	try:
		model.load_state_dict(ckpt['model_state'], strict=True)
	except RuntimeError as err:
		print("Runtime Error: {0}".format(err))
	except:
		print("Unexpected error:", sys.exc_info()[0])
		raise

	if args.cuda:
		device = get_freer_gpu()
		model = model.cuda(device)

	model.eval()

	correct = 0

	with torch.no_grad():

		iterator = tqdm(test_loader, total=len(test_loader))
		for batch in iterator:

			x, y = batch

			x = x.to(device)
			y = y.to(device)

			embeddings = model.forward(x)

			embeddings_norm = F.normalize(embeddings, p=2, dim=1)

			out = model.out_proj(embeddings_norm, y, eval_=True)

			pred = F.softmax(out, dim=1).max(1)[1].long()
			correct += pred.squeeze().eq(y.squeeze()).detach().sum().item()

	print('\nAccuracy of model {}: {}'.format(args.cp_path.split('/')[-1], 100.*correct/len(testset)))


# Copyright 2013 Mirantis Inc.
# Copyright 2014 Cloudbase Solutions Srl
#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.

import mock
import unittest

from cloudbaseinit.openstack.common import cfg
from cloudbaseinit.plugins.windows.userdataplugins import urldownload

CONF = cfg.CONF

class UrlDownloadHandlerTests(unittest.TestCase):

    def setUp(self):
        self._urldownload = urldownload.URLDownloadPlugin()

    @mock.patch('cloudbaseinit.plugins.windows.userdatautils'
                '.execute_user_data_script')

    def _test_process(self, mock_execute_user_data_script, filename):
        mock_part = mock.MagicMock()
        mock_part.get_filename.return_value = filename
        response = self._urldownload.process(mock_part)
        mock_part.get_filename.assert_called_with()
        if filename:
            mock_execute_user_data_script.assert_called_with(
                mock_part.get_payload())
            self.assertEqual(response, mock_execute_user_data_script())
        else:
            self.assertTrue(response is None)

        response = self._urldownload.process(mock_part)
        mock_part.get_filename.assert_called_with()
        if filename:
            mock_execute_user_data_script.assert_called_with(
                mock_part.get_payload())
            self.assertEqual(response, mock_execute_user_data_script())
        else:
            self.assertTrue(response is None)

    def test_process(self):
        self._test_process(filename='cfn-userdata')

    def test_process_content_not_supported(self):
        self._test_process(filename=None)


import pandas as pd
import numpy as np
from scipy import stats
import tensorflow as tf
from sklearn.model_selection import train_test_split

def windows(data, size):
    start = 0
    while start < data.count():
        yield int(start), int(start + size)
        start += (size / 2)

def segment_signal(data,window_size = 12):
    segments = np.empty((0,window_size,6))
    labels = np.empty((0))
    for (start, end) in windows(data['Date'], window_size):
        x = data["mav5"][start:end]
        y = data["mav10"][start:end]
        z = data["mav20"][start:end]
        a = data["mav30"][start:end]
        b = data["mav50"][start:end]
        c = data["mav100"][start:end]
        if(len(data['Date'][start:end]) == window_size):
            segments = np.vstack([segments,np.dstack([x,y,z,a,b,c])])
            labels = np.append(labels,stats.mode(data["Action"][start:end])[0][0])
    return segments, labels

def get_batches(X, y, batch_size = 100):
    """ Return a generator for batches """
    n_batches = len(X) // batch_size
    X, y = X[:n_batches*batch_size], y[:n_batches*batch_size]

    # Loop over batches and yield
    for b in range(0, len(X), batch_size):
        yield X[b:b+batch_size], y[b:b+batch_size]

def create_tensorflow_train_data(csvfilename):
    df = pd.read_csv('sampledata/'+ csvfilename)
    segments, labels = segment_signal(df)
    list_ch_train = df.Action.unique()
    labels = np.asarray(pd.get_dummies(labels), dtype = np.int8)
    X_tr, X_vld, lab_tr, lab_vld = train_test_split(segments, labels, stratify = labels, random_state = 123)

    return X_tr, X_vld, lab_tr, lab_vld, list_ch_train

def create_tensorflow_test_data(csvfilename):
    df = pd.read_csv('sampledata/'+ csvfilename)
    segments, labels = segment_signal(df)
    list_ch_test = df.Action.unique()
    labels = np.asarray(pd.get_dummies(labels), dtype = np.int8)
    X_test = segments
    y_test = labels

    return X_test, y_test, list_ch_test

Trading Rule Identification by CNN/tensorflow_setup.py


"""Extreme support."""
import time
import re
from netmiko.cisco_base_connection import CiscoSSHConnection

class ExtremeExosBase(CiscoSSHConnection):
    """Extreme Exos support.

    Designed for EXOS >= 15.0
    """

    def session_preparation(self):
        self._test_channel_read()
        self.set_base_prompt()
        self.disable_paging(command="disable clipaging")
        self.send_command_timing("disable cli prompting")
        # Clear the read buffer
        time.sleep(0.3 * self.global_delay_factor)
        self.clear_buffer()

    def set_base_prompt(self, *args, **kwargs):
        """
        Extreme attaches an id to the prompt. The id increases with every command.
        It needs to br stripped off to match the prompt. Eg.

            testhost.1 #
            testhost.2 #
            testhost.3 #

        If new config is loaded and not saved yet, a '* ' prefix appears before the
        prompt, eg.

            * testhost.4 #
            * testhost.5 #
        """
        cur_base_prompt = super().set_base_prompt(*args, **kwargs)
        # Strip off any leading * or whitespace chars; strip off trailing period and digits
        match = re.search(r"[\*\s]*(.*)\.\d+", cur_base_prompt)
        if match:
            self.base_prompt = match.group(1)
            return self.base_prompt
        else:
            return self.base_prompt

    def send_command(self, *args, **kwargs):
        """Extreme needs special handler here due to the prompt changes."""

        # Change send_command behavior to use self.base_prompt
        kwargs.setdefault("auto_find_prompt", False)

        # refresh self.base_prompt
        self.set_base_prompt()
        return super().send_command(*args, **kwargs)

    def config_mode(self, config_command=""):
        """No configuration mode on Extreme Exos."""
        return ""

    def check_config_mode(self, check_string="#"):
        """Checks whether in configuration mode. Returns a boolean."""
        return super().check_config_mode(check_string=check_string)

    def exit_config_mode(self, exit_config=""):
        """No configuration mode on Extreme Exos."""
        return ""

    def save_config(
        self, cmd="save configuration primary", confirm=False, confirm_response=""
    ):
        """Saves configuration."""
        return super().save_config(
            cmd=cmd, confirm=confirm, confirm_response=confirm_response
        )

class ExtremeExosSSH(ExtremeExosBase):
    pass

class ExtremeExosTelnet(ExtremeExosBase):
    def __init__(self, *args, **kwargs):
        default_enter = kwargs.get("default_enter")
        kwargs["default_enter"] = "\r\n" if default_enter is None else default_enter
        super().__init__(*args, **kwargs)


# -*- coding: utf-8 -*-
"""A model of Infrastructure Virtual Machines area of CFME.  This includes the VMs explorer tree,
quadicon lists, and VM details page.
"""
import fauxfactory
import re
from cfme import js
from cfme.exceptions import CandidateNotFound, VmNotFound, OptionNotAvailable, TemplateNotFound
from cfme.fixtures import pytest_selenium as sel
from cfme.services import requests
from cfme.web_ui import (
    CheckboxTree, Form, Region, Quadicon, Tree, accordion, fill, flash, form_buttons, paginator,
    toolbar, Calendar, Select, Input, mixins
)
from cfme.web_ui.menu import nav
from datetime import date
from contextlib import contextmanager
from functools import partial
from selenium.common.exceptions import NoSuchElementException
from utils.conf import cfme_data
from utils.log import logger
from utils.pretty import Pretty
from utils.timeutil import parsetime
from utils.virtual_machines import deploy_template
from utils.wait import wait_for, TimedOutError
from utils.mgmt_system import ActionNotSupported, VMInstanceNotFound
from utils import version

QUADICON_TITLE_LOCATOR = ("//div[@id='quadicon']/../../../tr/td/a[contains(@href,'vm_infra/x_show')"
                         " or contains(@href, '/show/')]")  # for provider specific vm/template page

details_page = Region(infoblock_type='detail')

cfg_btn = partial(toolbar.select, 'Configuration')
pol_btn = partial(toolbar.select, 'Policy')
lcl_btn = partial(toolbar.select, 'Lifecycle')
mon_btn = partial(toolbar.select, 'Monitoring')
pwr_btn = partial(toolbar.select, 'Power')

create_button = form_buttons.FormButton("Create")

visible_tree = Tree("//div[@class='dhxcont_global_content_area']"
                    "[not(contains(@style, 'display: none'))]/div/div/div"
                    "/ul[@class='dynatree-container']")

manage_policies_tree = CheckboxTree(
    {
        version.LOWEST: "//div[@id='treebox']/div/table",
        "5.3": "//div[@id='protect_treebox']/ul"
    }
)

manage_policies_page = Region(
    locators={
        'save_button': form_buttons.save,
    })

snapshot_tree = Tree(
    {
        version.LOWEST: "//div[@id='treebox']/div/table",
        "5.3": "//div[@id='snapshots_treebox']/ul"
    }
)

snapshot_form = Form(
    fields=[
        ('name', Input('name')),
        ('descrition', Input('description')),
        ('snapshot_memory', Input('snap_memory')),
        ('create_button', create_button),
        ('cancel_button', form_buttons.cancel)
    ])

def date_retire_element(fill_data):
    """We need to call this function that will mimic clicking the calendar, picking the date and
    the subsequent callbacks from the server"""
    # TODO: Move the code in the Calendar itself? I did not check other calendars
    if isinstance(fill_data, date):
        date_str = '%s/%s/%s' % (fill_data.month, fill_data.day, fill_data.year)
    else:
        date_str = str(fill_data)
    sel.execute_script(
        js.update_retirement_date_function_script +
        "updateDate(arguments[0]);",
        date_str
    )

retire_form = Form(fields=[
    ('date_retire', date_retire_element),
    ('warn', sel.Select("select#retirement_warn"))
])

retirement_date_form = Form(fields=[
    ('retirement_date_text', Calendar("miq_date_1")),
    ('retirement_warning_select', Select("//select[@id='retirement_warn']"))
])

retire_remove_button = "//span[@id='remove_button']/a/img"

set_ownership_form = Form(fields=[
    ('user_name', Select("//select[@id='user_name']")),
    ('group_name', Select("//select[@id='group_name']")),
    ('create_button', form_buttons.save),
    ('reset_button', form_buttons.reset),
    ('cancel_button', form_buttons.cancel)
])

nav.add_branch(
    'infrastructure_virtual_machines',
    {
        "infra_vm_and_templates":
        [
            lambda _: accordion.tree("VMs & Templates", "All VMs & Templates"),
            {
                "vm_templates_provider_branch":
                [
                    lambda ctx: visible_tree.click_path(ctx["provider_name"]),
                    {
                        "datacenter_branch":
                        [
                            lambda ctx: visible_tree.click_path(ctx["datacenter_name"]),
                            {
                                "infra_vm_obj": lambda ctx: visible_tree.click_path(ctx["vm_name"]),
                            }
                        ],
                    }
                ],

                "vm_templates_archived_branch":
                [
                    lambda ctx: visible_tree.click_path("<Archived>"),
                    {
                        "infra_archive_obj":
                        lambda ctx: visible_tree.click_path(ctx["archive_name"]),
                    }
                ],

                "vm_templates_orphaned_branch":
                [
                    lambda ctx: visible_tree.click_path('<Orphaned>'),
                    {
                        "infra_orphan_obj": lambda ctx: visible_tree.click_path(ctx["orphan_name"]),
                    }
                ],
            }
        ],

        "infra_vms":
        [
            lambda _: accordion.tree("VMs", "All VMs"),
            {
                "infra_vms_filter_folder":
                [
                    lambda ctx: visible_tree.click_path(ctx["folder_name"]),
                    {
                        "infra_vms_filter": lambda ctx: visible_tree.click_path(ctx["filter_name"]),
                    }
                ],

                "infra_vm_by_name": lambda ctx: sel.click(ctx['vm'].find_quadicon(
                    do_not_navigate=True))
            }
        ],

        "infra_templates":
        [
            lambda _: (accordion.tree("Templates", "All Templates"), toolbar.set_vms_grid_view()),
            {
                "infra_templates_filter_folder":
                [
                    lambda ctx: visible_tree.click_path(ctx["folder_name"]),
                    {
                        "infra_templates_filter":
                        lambda ctx: visible_tree.click_path(ctx["filter_name"]),
                    }
                ],
            }
        ],
    }
)

class Common(Pretty):

    pretty_attrs = ["name", "provider_crud", "template_name"]

    def __init__(self):
        self._assigned_pp = set([])

    def _load_details(self, refresh=False, is_vm=True):
        """Navigates to a VM's details page.

        Args:
            refresh: Refreshes the vm page if already there

        Raises:
            VmNotFound:
                When unable to find the VM passed
        """
        if not self.on_details(is_vm=is_vm):
            logger.debug("load_vm_details: not on details already")
            sel.click(self._find_quadicon(is_vm=is_vm))
        else:
            if refresh:
                toolbar.refresh()

    def on_details(self, force=False, is_vm=True):
        """A function to determine if the browser is already on the proper vm details page.
        """
        locator = ("//div[@class='dhtmlxInfoBarLabel' and contains(. , 'VM and Instance \"%s\"')]" %
                   self.name)
        if not sel.is_displayed(locator):
            if not force:
                return False
            else:
                self.load_details(is_vm=is_vm)
                return True

        text = sel.text(locator).encode("utf-8")
        pattern = r'("[A-Za-z0-9_\./\\-]*")'
        m = re.search(pattern, text)

        if not force:
            return self.name == m.group().replace('"', '')
        else:
            if self.name != m.group().replace('"', ''):
                self._load_details(is_vm=is_vm)
            else:
                return False

    def _find_quadicon(
            self, is_vm=True, do_not_navigate=False, mark=False, refresh=True,
            from_any_provider=False):
        """Find and return a quadicon belonging to a specific vm

        Args:
            from_any_provider: Whether to look for it anywhere (root of the tree). Useful when
                looking up archived or orphaned VMs

        Returns: :py:class:`cfme.web_ui.Quadicon` instance
        Raises: VmNotFound
        """
        quadicon = Quadicon(self.name, "vm")
        if not do_not_navigate:
            if from_any_provider:
                if is_vm:
                    sel.force_navigate("infra_vms")
                else:
                    sel.force_navigate("infra_templates")
            elif is_vm:
                self.provider_crud.load_all_provider_vms()
            else:
                self.provider_crud.load_all_provider_templates()
            toolbar.set_vms_grid_view()
        elif refresh:
            sel.refresh()
        if not paginator.page_controls_exist():
            if is_vm:
                raise VmNotFound("VM '{}' not found in UI!".format(self.name))
            else:
                raise TemplateNotFound("Template '{}' not found in UI!".format(self.name))

        paginator.results_per_page(1000)
        for page in paginator.pages():
            if sel.is_displayed(quadicon, move_to=True):
                if mark:
                    sel.check(quadicon.checkbox())
                return quadicon
        else:
            raise VmNotFound("VM '{}' not found in UI!".format(self.name))

    def does_vm_exist_on_provider(self):
        """Check if VM exists on provider itself"""
        return self.provider_crud.get_mgmt_system().does_vm_exist(self.name)

    def _method_helper(self, from_details=False):
        if from_details:
            self.load_details()
        else:
            self.find_quadicon(mark=True)

    def _remove_from_cfme(self, is_vm=True, cancel=True, from_details=False):
        """Removes a VM from CFME VMDB

        Args:
            cancel: Whether to cancel the deletion, defaults to True
            from_details: whether to delete from the details page
        """
        self._method_helper(from_details)
        if from_details:
            cfg_btn('Remove from the VMDB', invokes_alert=True)
        elif not is_vm:
            cfg_btn('Remove Templates from the VMDB', invokes_alert=True)
        else:
            cfg_btn('Remove selected items from the VMDB', invokes_alert=True)
        sel.handle_alert(cancel=cancel)

    def delete_from_provider(self):
        provider_mgmt = self.provider_crud.get_mgmt_system()
        if provider_mgmt.does_vm_exist(self.name):
            try:
                if provider_mgmt.is_vm_suspended(self.name):
                    logger.debug("Powering up VM %s to shut it down correctly on %s." %
                                 (self.name, self.provider_crud.key))
                    provider_mgmt.start_vm(self.name)
            except ActionNotSupported:
                # Action is not supported on mgmt system. Simply continue
                pass
            # One more check (for the suspended one)
            if provider_mgmt.does_vm_exist(self.name):
                try:
                    return provider_mgmt.delete_vm(self.name)
                except VMInstanceNotFound:
                    # Does not exist already
                    return True
        else:
            return True

    def get_detail(self, properties=None, icon_href=False):
        """Gets details from the details infoblock

        The function first ensures that we are on the detail page for the specific vm.

        Args:
            properties: An InfoBlock title, followed by the Key name, e.g. "Relationships", "Images"

        Returns:
            A string representing the contents of the InfoBlock's value.
        """
        self.load_details(refresh=True)
        if icon_href:
            return details_page.infoblock.icon_href(*properties)
        else:
            return details_page.infoblock.text(*properties)

    @property
    def compliance_status(self):
        """Returns the title of the compliance infoblock. The title contains datetime so it can be
        compared.

        Returns:
            :py:class:`NoneType` if no title is present (no compliance checks before), otherwise str
        """
        self.load_details(refresh=True)
        return details_page.infoblock("Compliance", "Status").title

    @property
    def compliant(self):
        """Check if the VM is compliant

        Returns:
            :py:class:`NoneType` if the VM was never verified, otherwise :py:class:`bool`
        """
        text = self.get_detail(properties=("Compliance", "Status")).strip().lower()
        if text == "never verified":
            return None
        elif text.startswith("non-compliant"):
            return False
        elif text.startswith("compliant"):
            return True
        else:
            raise ValueError("{} is not a known state for compliance".format(text))

    @contextmanager
    def check_compliance_wrapper(self, timeout=240):
        """This wrapper takes care of waiting for the compliance status to change

        Args:
            timeout: Wait timeout in seconds.
        """
        self.load_details(refresh=True)
        original_state = self.compliance_status
        yield
        wait_for(
            lambda: self.compliance_status != original_state,
            num_sec=timeout, delay=5, message="compliance of {} checked".format(self.name),
            fail_func=lambda: toolbar.select("Reload"))

    def check_compliance_and_wait(self, timeout=240):
        with self.check_compliance_wrapper(timeout=timeout):
            self.check_compliance()
        return self.compliant

    def rediscover(self):
        """Deletes the VM from the provider and lets it discover again"""
        self.remove_from_cfme(cancel=False, from_details=True)
        wait_for(self.does_vm_exist_in_cfme, fail_condition=True,
                 num_sec=300, delay=15, fail_func=sel.refresh)
        self.provider_crud.refresh_provider_relationships()
        self.wait_to_appear()

    def rediscover_if_analysis_data_present(self):
        if self.get_detail(properties=('Lifecycle', 'Last Analyzed')).lower() != 'never':
            self.rediscover()
            return True
        return False

    def get_tags(self, tag="My Company Tags"):
        """Returns all tags that are associated with this VM"""
        self.load_details(refresh=True)
        return mixins.get_tags(tag)

    def refresh_relationships(self, from_details=False, cancel=False):
        """Executes a refresh relationships action against a list of VMs.

        Args:
            from_details: Whether or not to perform action from vm details page
            cancel: Whether or not to cancel the refresh relationships action
        """
        if from_details:
            self.load_details()
        else:
            self.find_quadicon(mark=True)
        cfg_btn('Refresh Relationships and Power States', invokes_alert=True)
        sel.handle_alert(cancel=cancel)

    def smartstate_scan(self, cancel=True, from_details=False):
        self._method_helper(from_details=from_details)
        cfg_btn('Perform SmartState Analysis', invokes_alert=True)
        sel.handle_alert(cancel=cancel)

    def wait_to_appear(self, timeout=600, load_details=True):
        """Wait for a VM to appear within CFME

        Args:
            timeout: time (in seconds) to wait for it to appear
            from_details: when found, should it load the vm details
        """
        wait_for(self.does_vm_exist_in_cfme, num_sec=timeout, delay=30, fail_func=sel.refresh)
        if load_details:
            self.load_details()

    def wait_for_delete(self, timeout=600):
        wait_for(self.does_vm_exist_in_cfme, num_sec=timeout, delay=30, fail_func=sel.refresh,
                 fail_condition=True)

    def does_vm_exist_in_cfme(self):
        """A function to tell you if a VM exists or not.
        """
        try:
            self.find_quadicon()
            return True
        except VmNotFound:
            return False

    @property
    def genealogy(self):
        return Genealogy(self)

    def check_compliance(self):
        self.load_details(refresh=True)
        pol_btn("Check Compliance of Last Known Configuration", invokes_alert=True)
        sel.handle_alert()
        flash.assert_no_errors()

    def assign_policy_profiles(self, *policy_profile_names):
        """ Assign Policy Profiles to this VM.

        Args:
            policy_profile_names: :py:class:`str` with Policy Profile names. After Control/Explorer
                coverage goes in, PolicyProfile objects will be also passable.
        """
        map(self._assigned_pp.add, policy_profile_names)
        self._assign_unassign_policy_profiles(True, *policy_profile_names)

    def unassign_policy_profiles(self, *policy_profile_names):
        """ Unssign Policy Profiles to this VM.

        Args:
            policy_profile_names: :py:class:`str` with Policy Profile names. After Control/Explorer
                coverage goes in, PolicyProfile objects will be also passable.
        """
        for pp_name in policy_profile_names:
            try:
                self._assigned_pp.remove(pp_name)
            except KeyError:
                pass
        self._assign_unassign_policy_profiles(False, *policy_profile_names)

    def _assign_unassign_policy_profiles(self, assign, *policy_profile_names):
        """DRY function for managing policy profiles.

        See :py:func:`assign_policy_profiles` and :py:func:`assign_policy_profiles`

        Args:
            assign: Wheter to assign or unassign.
            policy_profile_names: :py:class:`str` with Policy Profile names.
        """
        self.load_details(refresh=True)
        pol_btn("Manage Policies")
        for policy_profile in policy_profile_names:
            if assign:
                manage_policies_tree.check_node(policy_profile)
            else:
                manage_policies_tree.uncheck_node(policy_profile)
        sel.move_to_element('#tP')
        form_buttons.save()

    def set_ownership(self, user=None, group=None, click_cancel=False, click_reset=False):
        sel.click(self.find_quadicon(False, False, False))
        cfg_btn('Set Ownership')
        if click_reset:
            action = form_buttons.reset
            msg_assert = lambda: flash.assert_message_match(
                'All changes have been reset')
        elif click_cancel:
            action = form_buttons.cancel
            msg_assert = lambda: flash.assert_success_message(
                'Set Ownership was cancelled by the user')
        else:
            action = form_buttons.save
            msg_assert = lambda: flash.assert_success_message(
                'Ownership saved for selected Virtual Machine')
        fill(set_ownership_form, {'user_name': user, 'group_name': group},
             action=action)
        msg_assert()

    def unset_ownership(self):
        # choose the vm code comes here
        sel.click(self.find_quadicon(False, False, False))
        cfg_btn('Set Ownership')
        fill(set_ownership_form, {'user_name': '<No Owner>', 'group_name': '<No Group>'},
            action=form_buttons.save)
        flash.assert_success_message('Ownership saved for selected Virtual Machine')

class Vm(Common):
    """Represents a VM in CFME

    Args:
        name: Name of the VM
        provider_crud: :py:class:`cfme.cloud.provider.Provider` object
        template_name: Name of the template to use for provisioning
    """

    class Snapshot(object):
        def __init__(self, name=None, description=None, memory=None, parent_vm=None):
            super(Vm.Snapshot, self).__init__()
            self.name = name
            self.description = description
            self.memory = memory
            self.vm = parent_vm

        def _nav_to_snapshot_mgmt(self):
            locator = ("//div[@class='dhtmlxInfoBarLabel' and " +
                       "contains(. , '\"Snapshots\" for Virtual Machine \"%s\"') ]" % self.name)
            if not sel.is_displayed(locator):
                self.vm.load_details()
                sel.click(details_page.infoblock.element("Properties", "Snapshots"))

        def does_snapshot_exist(self):
            self._nav_to_snapshot_mgmt()
            try:
                snapshot_tree.find_path_to(re.compile(r".*?\(Active\)$"))
                return True
            except CandidateNotFound:
                try:
                    snapshot_tree.find_path_to(re.compile(self.name))
                    return True
                except CandidateNotFound:
                    return False
            except NoSuchElementException:
                return False

        def wait_for_snapshot_active(self):
            self._nav_to_snapshot_mgmt()
            try:
                snapshot_tree.click_path(*snapshot_tree.find_path_to(re.compile(self.name)))
                if sel.is_displayed_text(self.name + " (Active)"):
                    return True
            except CandidateNotFound:
                return False

        def create(self):
            self._nav_to_snapshot_mgmt()
            toolbar.select('Create a new snapshot for this VM')
            fill(snapshot_form, {'name': self.name,
                                 'description': self.description,
                                 'snapshot_memory': self.memory
                                 },
                 action=snapshot_form.create_button)
            wait_for(self.does_snapshot_exist, num_sec=300, delay=20, fail_func=sel.refresh)

        def delete(self, cancel=False):
            self._nav_to_snapshot_mgmt()
            toolbar.select('Delete Snapshots', 'Delete Selected Snapshot', invokes_alert=True)
            sel.handle_alert(cancel=cancel)
            if not cancel:
                flash.assert_message_match(
                    'Delete Snapshot initiated for 1 VM and Instance from the CFME Database')

        def delete_all(self, cancel=False):
            self._nav_to_snapshot_mgmt()
            toolbar.select('Delete Snapshots', 'Delete All Existing Snapshots', invokes_alert=True)
            sel.handle_alert(cancel=cancel)
            if not cancel:
                flash.assert_message_match(
                    'Delete All Snapshots initiated for 1 VM and Instance from the CFME Database')

        def revert_to(self, cancel=False):
            self._nav_to_snapshot_mgmt()
            snapshot_tree.click_path(*snapshot_tree.find_path_to(re.compile(self.name)))
            toolbar.select('Revert to selected snapshot', invokes_alert=True)
            sel.handle_alert(cancel=cancel)
            flash.assert_message_match(
                'Revert To A Snapshot initiated for 1 VM and Instance from the CFME Database')

    # POWER CONTROL OPTIONS
    SUSPEND = "Suspend"
    POWER_ON = "Power On"
    POWER_OFF = "Power Off"
    GUEST_RESTART = "Restart Guest"
    GUEST_SHUTDOWN = "Shutdown Guest"
    RESET = "Reset"
    # POWER STATE
    STATE_ON = "on"
    STATE_OFF = "off"
    STATE_SUSPENDED = "suspended"

    def __init__(self, name, provider_crud, template_name=None):
        super(Vm, self).__init__()
        self.name = name
        self.template_name = template_name
        self.provider_crud = provider_crud

    def create_on_provider(self, timeout=900, find_in_cfme=False, **kwargs):
        """Create the VM on the provider

        Args:
            timeout: Number of seconds to wait for the VM to appear in CFME
                     Will not wait at all, if set to 0 (Defaults to ``900``)
        """
        deploy_template(self.provider_crud.key, self.name, self.template_name, **kwargs)
        if find_in_cfme:
            self.provider_crud.refresh_provider_relationships()
            self.wait_to_appear(timeout=timeout, load_details=False)

    def load_details(self, refresh=False):
        self._load_details(is_vm=True, refresh=refresh)

    def open_edit(self):
        self.load_details(refresh=True)
        toolbar.select("Configuration", "Edit this VM")

    def find_quadicon(self, do_not_navigate=False, mark=False, refresh=True, **kwargs):
        return self._find_quadicon(
            is_vm=True, do_not_navigate=do_not_navigate, mark=mark, refresh=refresh, **kwargs)

    def remove_from_cfme(self, cancel=False, from_details=False):
        """Removes a VM from CFME VMDB"""
        self._remove_from_cfme(is_vm=True, cancel=cancel, from_details=from_details)

    def power_control_from_provider(self, option):
        """Power control a vm from the provider

        Args:
            option: power control action to take against vm

        Raises:
            OptionNotAvailable: option parm must have proper value
        """
        if option == Vm.POWER_ON:
            self.provider_crud.get_mgmt_system().start_vm(self.name)
        elif option == Vm.POWER_OFF:
            self.provider_crud.get_mgmt_system().stop_vm(self.name)
        elif option == Vm.SUSPEND:
            self.provider_crud.get_mgmt_system().suspend_vm(self.name)
        # elif reset:
        # elif shutdown:
        else:
            raise OptionNotAvailable(option + " is not a supported action")

    def power_control_from_cfme(self, option, cancel=True, from_details=False):
        """Power controls a VM from within CFME

        Args:
            option: corresponds to option values under the power button
            cancel: Whether or not to cancel the power operation on confirmation
            from_details: Whether or not to perform action from vm details page
        """
        if (self.is_pwr_option_available_in_cfme(option=option, from_details=from_details)):
                pwr_btn(option, invokes_alert=True)
                sel.handle_alert(cancel=cancel)
                logger.info(
                    "Power control action of vm %s, option %s, cancel %s executed" %
                    (self.name, option, str(cancel)))
        else:
            raise OptionNotAvailable(option + " is not visible or enabled")

    def is_pwr_option_available_in_cfme(self, option, from_details=False):
        """Checks to see if a power option is available on the VM

        Args:
            option: corresponds to option values under the power button, preferred approach
                is to use Vm option constansts
            from_details: Whether or not to perform action from vm details page
        """
        self._method_helper(from_details=from_details)
        try:
            return not toolbar.is_greyed('Power', option)
        except NoSuchElementException:
            return False

    def retire(self):
        sel.force_navigate("infra_vm_by_name", context={'vm': self})
        lcl_btn("Retire this VM", invokes_alert=True)
        sel.handle_alert()
        flash.assert_success_message(
            "Retire initiated for 1 VM and Instance from the CFME Database")

    def migrate_vm(self, email=None, first_name=None, last_name=None,
                   host_name=None, datastore_name=None):
        sel.force_navigate("infra_vm_by_name", context={'vm': self})
        lcl_btn("Migrate this VM")
        first_name = first_name or fauxfactory.gen_alphanumeric()
        last_name = last_name or fauxfactory.gen_alphanumeric()
        email = email or "{}@{}.test".format(first_name, last_name)
        try:
            prov_data = cfme_data["management_systems"][self.provider_crud.key]["provisioning"]
        except (KeyError, IndexError):
            raise ValueError("You have to specify the correct options in cfme_data.yaml")
        provisioning_data = {
            "first_name": first_name,
            "last_name": last_name,
            "email": email,
            "host_name": {"name": prov_data.get("host")},
            "datastore_name": {"name": prov_data.get("datastore")}
        }
        from cfme.provisioning import provisioning_form
        fill(provisioning_form, provisioning_data, action=provisioning_form.submit_button)

    def clone_vm(self, email=None, first_name=None, last_name=None,
                 vm_name=None, provision_type=None):
        sel.force_navigate("infra_vm_by_name", context={'vm': self})
        lcl_btn("Clone this VM")
        first_name = first_name or fauxfactory.gen_alphanumeric()
        last_name = last_name or fauxfactory.gen_alphanumeric()
        email = email or "{}@{}.test".format(first_name, last_name)
        try:
            prov_data = cfme_data["management_systems"][self.provider_crud.key]["provisioning"]
        except (KeyError, IndexError):
            raise ValueError("You have to specify the correct options in cfme_data.yaml")
        provisioning_data = {
            "first_name": first_name,
            "last_name": last_name,
            "email": email,
            "provision_type": provision_type,
            "vm_name": vm_name,
            "host_name": {"name": prov_data.get("host")},
            "datastore_name": {"name": prov_data.get("datastore")},
            "vlan": prov_data.get("vlan")
        }
        from cfme.provisioning import provisioning_form
        fill(provisioning_form, provisioning_data, action=provisioning_form.submit_button)

    @property
    def retirement_date(self):
        """Returns the retirement date of the selected machine.

        Returns:
            :py:class:`NoneType` if there is none, or :py:class:`utils.timeutil.parsetime`
        """
        date_str = self.get_detail(properties=("Lifecycle", "Retirement Date")).strip()
        if date_str.lower() == "never":
            return None
        return parsetime.from_american_date_only(date_str)

    def set_retirement_date(self, when, warn=None):
        """Sets the retirement date for this Vm object.

        It incorporates some magic to make it work reliably since the retirement form is not very
        pretty and it can't be just "done".

        Args:
            when: When to retire. :py:class:`str` in format mm/dd/yy of
                :py:class:`datetime.datetime` or :py:class:`utils.timeutil.parsetime`.
            warn: When to warn, fills the select in the form in case the ``when`` is specified.
        """
        self.load_details()
        lcl_btn("Set Retirement Date")
        sel.wait_for_element("#miq_date_1")
        if when is None:
            try:
                wait_for(lambda: sel.is_displayed(retire_remove_button), num_sec=5, delay=0.2)
                sel.click(retire_remove_button)
                wait_for(lambda: not sel.is_displayed(retire_remove_button), num_sec=10, delay=0.2)
                sel.click(form_buttons.save)
            except TimedOutError:
                pass
        else:
            if sel.is_displayed(retire_remove_button):
                sel.click(retire_remove_button)
                wait_for(lambda: not sel.is_displayed(retire_remove_button), num_sec=15, delay=0.2)
            fill(retire_form.date_retire, when)
            wait_for(lambda: sel.is_displayed(retire_remove_button), num_sec=15, delay=0.2)
            if warn is not None:
                fill(retire_form.warn, warn)
            sel.click(form_buttons.save)

    def publish_to_template(self, template_name, email=None, first_name=None, last_name=None):
        self.load_details()
        lcl_btn("Publish this VM to a Template")
        first_name = first_name or fauxfactory.gen_alphanumeric()
        last_name = last_name or fauxfactory.gen_alphanumeric()
        email = email or "{}@{}.test".format(first_name, last_name)
        try:
            prov_data = cfme_data["management_systems"][self.provider_crud.key]["provisioning"]
        except (KeyError, IndexError):
            raise ValueError("You have to specify the correct options in cfme_data.yaml")

        provisioning_data = {
            "first_name": first_name,
            "last_name": last_name,
            "email": email,
            "vm_name": template_name,
            "host_name": {"name": prov_data.get("host")},
            "datastore_name": {"name": prov_data.get("datastore")},
        }
        from cfme.provisioning import provisioning_form
        fill(provisioning_form, provisioning_data, action=provisioning_form.submit_button)
        cells = {'Description': 'Publish from [%s] to [%s]' % (self.name, template_name)}
        row, __ = wait_for(
            requests.wait_for_request, [cells], fail_func=requests.reload, num_sec=900, delay=20)
        return Template(template_name, self.provider_crud)

    def wait_for_vm_state_change(
            self, desired_state=None, timeout=300, from_details=False):
        """Wait for VM to come to desired state.

        This function waits just the needed amount of time thanks to wait_for.

        Args:
            desired_state: on, off, suspended... corresponds to values in cfme, preferred approach
                is to use Vm.STATE_* constansts
            timeout: Specify amount of time (in seconds) to wait
        Raises:
            TimedOutError:
                When VM does not come up to desired state in specified period of time.
            NoVmFound:
                When unable to find the VM passed
        """
        def _looking_for_state_change():
            if from_details:
                self.load_details(refresh=True)
                detail_t = ("Power Management", "Power State")
                return self.get_detail(properties=detail_t) == desired_state
            else:
                return self.find_quadicon().state == 'currentstate-' + desired_state
        return wait_for(_looking_for_state_change, num_sec=timeout, delay=30)

    class CfmeRelationship(object):

        relationship_form = Form(
            fields=[
                ('server_select', Select("//*[@id='server_id']")),
                ('save_button', form_buttons.save),
                ('reset_button', form_buttons.reset),
                ('cancel_button', form_buttons.cancel)
            ])

        def __init__(self, o):
            self.o = o

        def navigate(self):
            self.o.load_details()
            cfg_btn('Edit Management Engine Relationship')

        def is_relationship_set(self):
            return "<Not a Server>" not in self.get_relationship()

        def get_relationship(self):
            self.navigate()
            rel = str(self.relationship_form.server_select.all_selected_options[0].text)
            form_buttons.cancel()
            return rel

        def set_relationship(self, server_name, server_id, click_cancel=False):
            self.navigate()
            option = "%s (%d)" % (server_name, server_id)

            if click_cancel:
                fill(self.relationship_form, {'server_select': option},
                     action=self.relationship_form.cancel_button)
            else:
                fill(self.relationship_form, {'server_select': option},
                     action=self.relationship_form.save_button)
                # something weird going on where changing the select doesn't POST to undim save
                sel.wait_for_ajax()
                if self.relationship_form.save_button.is_dimmed:
                    logger.warning("Worked around dimmed save button")
                    sel.browser().execute_script(
                        "$j.ajax({type: 'POST', url: '/vm_infra/evm_relationship_field_changed',"
                        " data: {'server_id':'%s'}})" % (server_id))
                    sel.click(form_buttons.FormButton(
                        "Save Changes", dimmed_alt="Save", force_click=True))
                flash.assert_success_message("Management Engine Relationship saved")

class Template(Common):

    def __init__(self, name, provider_crud):
        super(Template, self).__init__()
        self.name = name
        self.template_name = name
        self.provider_crud = provider_crud

    def load_details(self, refresh=False):
        self._load_details(refresh=refresh, is_vm=False)

    def find_quadicon(self, do_not_navigate=False, mark=False, refresh=True, **kwargs):
        return self._find_quadicon(
            is_vm=False, do_not_navigate=do_not_navigate, mark=mark, refresh=refresh, **kwargs)

    def remove_from_cfme(self, cancel=False, from_details=False):
        """Removes a VM from CFME VMDB"""
        self._remove_from_cfme(is_vm=False, cancel=cancel, from_details=from_details)

class Genealogy(object):
    """Class, representing genealogy of an infra object with possibility of data retrieval
    and comparison.

    Args:
        o: The :py:class:`Vm` or :py:class:`Template` object.
    """
    genealogy_tree = CheckboxTree({
        version.LOWEST: "//div[@id='treebox']/div/table",
        "5.3": "//div[@id='genealogy_treebox']/ul",
    })

    section_comparison_tree = CheckboxTree("//div[@id='all_sections_treebox']/div/table")
    apply_button = form_buttons.FormButton("Apply sections")

    mode_mapping = {
        "exists": "Exists Mode",
        "details": "Details Mode",
    }

    attr_mapping = {
        "all": "All Attributes",
        "different": "Attributes with different values",
        "same": "Attributes with same values",
    }

    def __init__(self, o):
        self.o = o

    def navigate(self):
        self.o.load_details()
        sel.click(details_page.infoblock.element("Relationships", "Genealogy"))

    def compare(self, *objects, **kwargs):
        """Compares two or more objects in the genealogy.

        Args:
            *objects: :py:class:`Vm` or :py:class:`Template` or :py:class:`str` with name.

        Keywords:
            sections: Which sections to compare.
            attributes: `all`, `different` or `same`. Default: `all`.
            mode: `exists` or `details`. Default: `exists`."""
        sections = kwargs.get("sections", None)
        attributes = kwargs.get("attributes", "all").lower()
        mode = kwargs.get("mode", "exists").lower()
        assert len(objects) >= 2, "You must specify at least two objects"
        objects = map(lambda o: o.name if isinstance(o, (Vm, Template)) else o, objects)
        self.navigate()
        for obj in objects:
            if not isinstance(obj, list):
                path = self.genealogy_tree.find_path_to(obj)
            self.genealogy_tree.check_node(*path)
        toolbar.select("Compare selected VMs")
        # COMPARE PAGE
        flash.assert_no_errors()
        if sections is not None:
            map(lambda path: self.section_comparison_tree.check_node(*path), sections)
            sel.click(self.apply_button)
            flash.assert_no_errors()
        # Set requested attributes sets
        toolbar.select(self.attr_mapping[attributes])
        # Set the requested mode
        toolbar.select(self.mode_mapping[mode])

    @property
    def tree(self):
        """Returns contents of the tree with genealogy"""
        self.navigate()
        return self.genealogy_tree.read_contents()

    @property
    def ancestors(self):
        """Returns list of ancestors of the represented object."""
        self.navigate()
        path = self.genealogy_tree.find_path_to(re.compile(r"^.*?\(Selected\)$"))
        if not path:
            raise ValueError("Something wrong happened, path not found!")
        processed_path = []
        for step in path[:-1]:
            # We will remove the (parent) and (Selected) suffixes
            processed_path.append(re.sub(r"\s*(?:\(Current\)|\(Parent\))$", "", step))
        return processed_path

def _method_setup(vm_names, provider_crud=None):
    """ Reduces some redundant code shared between methods """
    if isinstance(vm_names, basestring):
        vm_names = [vm_names]

    if provider_crud:
        provider_crud.load_all_provider_vms()
    else:
        sel.force_navigate('infra_vms')
    if paginator.page_controls_exist():
        paginator.results_per_page(1000)
    for vm_name in vm_names:
        sel.check(Quadicon(vm_name, 'vm').checkbox())

def find_quadicon(vm_name, do_not_navigate=False):
    """Find and return a quadicon belonging to a specific vm

    Args:
        vm: vm name as displayed at the quadicon
    Returns: :py:class:`cfme.web_ui.Quadicon` instance
    """
    if not do_not_navigate:
        sel.force_navigate('infra_vms')
    if not paginator.page_controls_exist():
        raise VmNotFound("VM '{}' not found in UI!".format(vm_name))

    paginator.results_per_page(1000)
    for page in paginator.pages():
        quadicon = Quadicon(vm_name, "vm")
        if sel.is_displayed(quadicon):
            return quadicon
    else:
        raise VmNotFound("VM '{}' not found in UI!".format(vm_name))

def remove(vm_names, cancel=True, provider_crud=None):
    """Removes multiple VMs from CFME VMDB

    Args:
        vm_names: List of VMs to interact with
        cancel: Whether to cancel the deletion, defaults to True
        provider_crud: provider object where vm resides on (optional)
    """
    _method_setup(vm_names, provider_crud)
    cfg_btn('Remove selected items from the VMDB', invokes_alert=True)
    sel.handle_alert(cancel=cancel)

def wait_for_vm_state_change(vm_name, desired_state, timeout=300, provider_crud=None):
    """Wait for VM to come to desired state.

    This function waits just the needed amount of time thanks to wait_for.

    Args:
        vm_name: Displayed name of the VM
        desired_state: 'on' or 'off'
        timeout: Specify amount of time (in seconds) to wait until TimedOutError is raised
        provider_crud: provider object where vm resides on (optional)
    """
    def _looking_for_state_change():
        toolbar.refresh()
        find_quadicon(vm_name, do_not_navigate=False).state == 'currentstate-' + desired_state

    _method_setup(vm_name, provider_crud)
    return wait_for(_looking_for_state_change, num_sec=timeout)

def is_pwr_option_visible(vm_names, option, provider_crud=None):
    """Returns whether a particular power option is visible.

    Args:
        vm_names: List of VMs to interact with, if from_details=True is passed, only one VM can
            be passed in the list.
        option: Power option param.
        provider_crud: provider object where vm resides on (optional)
    """
    _method_setup(vm_names, provider_crud)
    try:
        toolbar.is_greyed('Power', option)
        return True
    except NoSuchElementException:
        return False

def is_pwr_option_enabled(vm_names, option, provider_crud=None):
    """Returns whether a particular power option is enabled.

    Args:
        vm_names: List of VMs to interact with
        provider_crud: provider object where vm resides on (optional)
        option: Power option param.

    Raises:
        NoOptionAvailable:
            When unable to find the power option passed
    """
    _method_setup(vm_names, provider_crud)
    try:
        return not toolbar.is_greyed('Power', option)
    except NoSuchElementException:
        raise OptionNotAvailable("No such power option (" + str(option) + ") is available")

def do_power_control(vm_names, option, provider_crud=None, cancel=True):
    """Executes a power option against a list of VMs.

    Args:
        vm_names: List of VMs to interact with
        option: Power option param.
        provider_crud: provider object where vm resides on (optional)
        cancel: Whether or not to cancel the power control action
    """
    _method_setup(vm_names, provider_crud)

    if (is_pwr_option_visible(vm_names, provider_crud=provider_crud, option=option) and
            is_pwr_option_enabled(vm_names, provider_crud=provider_crud, option=option)):
                pwr_btn(option, invokes_alert=True)
                sel.handle_alert(cancel=cancel)

def refresh_relationships(vm_names, provider_crud=None, cancel=True):
    """Executes a refresh relationships action against a list of VMs.

    Args:
        vm_names: List of VMs to interact with
        provider_crud: provider object where vm resides on (optional)
        cancel: Whether or not to cancel the refresh relationships action
    """
    _method_setup(vm_names, provider_crud)
    cfg_btn('Refresh Relationships and Power States', invokes_alert=True)
    sel.handle_alert(cancel=cancel)

def perform_smartstate_analysis(vm_names, provider_crud=None, cancel=True):
    """Executes a refresh relationships action against a list of VMs.

    Args:
        vm_names: List of VMs to interact with
        provider_crud: provider object where vm resides on (optional)
        cancel: Whether or not to cancel the refresh relationships action
    """
    _method_setup(vm_names, provider_crud)
    cfg_btn('Perform SmartState Analysis', invokes_alert=True)
    sel.handle_alert(cancel=cancel)

def get_first_vm_title(do_not_navigate=False):
    if not do_not_navigate:
        sel.force_navigate('infra_vms')
    return Quadicon.get_first_quad_title()

def get_all_vms(do_not_navigate=False):
    """Returns list of all vms"""
    if not do_not_navigate:
        sel.force_navigate('infra_vms')
    vms = set([])
    if not paginator.page_controls_exist():
        return vms

    paginator.results_per_page(1000)
    for page in paginator.pages():
        try:
            for page in paginator.pages():
                for title in sel.elements(QUADICON_TITLE_LOCATOR):
                    vms.add(sel.get_attribute(title, "title"))
        except sel.NoSuchElementException:
            pass
    return vms

def get_number_of_vms(do_not_navigate=False):
    """
    Returns the total number of VMs visible to the user,
    including those archived or orphaned
    """
    logger.info("Getting number of vms")
    if not do_not_navigate:
        sel.force_navigate('infra_vms')
    if not paginator.page_controls_exist():
        logger.debug("No page controls")
        return 0
    total = paginator.rec_total()
    logger.debug("Number of VMs: {}".format(total))
    return int(total)

def _assign_unassign_policy_profiles(vm_name, assign, *policy_profile_names, **kwargs):
    """DRY function for managing policy profiles.

    See :py:func:`assign_policy_profiles` and :py:func:`assign_policy_profiles`

    Args:
        vm_name: Name of the VM.
        assign: Wheter to assign or unassign.
        policy_profile_names: :py:class:`str` with Policy Profile names.
    """
    if kwargs.get("via_details", True):
        sel.click(find_quadicon(vm_name))
    else:
        _method_setup(vm_name, **kwargs)
    toolbar.select("Policy", "Manage Policies")
    for policy_profile in policy_profile_names:
        if assign:
            manage_policies_tree.check_node(policy_profile)
        else:
            manage_policies_tree.uncheck_node(policy_profile)
    sel.click(form_buttons.save)

def assign_policy_profiles(vm_name, *policy_profile_names, **kwargs):
    """Assign Policy Profiles to specified VM.

    Args:
        vm_name: Name of the VM.
        policy_profile_names: :py:class:`str` with Policy Profile names.
    """
    return _assign_unassign_policy_profiles(vm_name, True, *policy_profile_names, **kwargs)

def unassign_policy_profiles(vm_name, *policy_profile_names, **kwargs):
    """Unassign Policy Profiles to specified VM.

    Args:
        vm_name: Name of the VM.
        policy_profile_names: :py:class:`str` with Policy Profile names.
    """
    return _assign_unassign_policy_profiles(vm_name, False, *policy_profile_names, **kwargs)


from testdata.library import base_library
import csv

class CreateAccountDt:
    def __init__(self, username):
        self.username = username

def get_csv_data(file_name, cre):
    file_path = base_library.create_file_path_input(file_name)
    rows = []
    csv_data = open(str(file_path))
    content = csv.reader(csv_data)

    # skip header line
    next(content, None)

    # add rows to list
    for row in content:
        rows.append(row)
    # convert row to object
    objs = []
    for r in rows:
        objs.append(cre(r[0]))

    return objs

def get_data(file_name):
    file_path = base_library.create_file_path_input(file_name)
    # create an empty list to store rows
    rows = []
    # open the CSV file
    data_file = open(str(file_path))
    # create a CSV Reader from CSV file
    reader = csv.reader(data_file)
    # skip the headers
    next(reader, None)
    # add rows from reader to list
    for row in reader:
        rows.append(row)
    return rows


# Import the required libraries
import pandas as pd
import numpy as np
import os
import glob
import json
import csv
import pyarrow as pa
from pyarrow import Table
import pyarrow.parquet as pq
import pyarrow.csv as pv
import pandasql as ps

#Folder path of source file
path=r'dataset'
os.listdir(path)
# List all files in a directory using os.listdir
json_pattern = os.path.join(path, '*.json')
file_list = glob.glob(json_pattern)

for fs in file_list:
    print(fs)
    df= pd.read_json(fs)
    #print(df)

#define the schema

def get_list_of_json_files():
    path=r'dataset'
    os.listdir(path)
    # List all files in a directory using os.listdir
    json_pattern = os.path.join(path, '*.json')
    list_of_files = glob.glob(json_pattern)
    return list_of_files

def create_list_from_json(jsonfile):
    with open(jsonfile) as f:
        data = json.load(f)
        #print(data)
        df2= pd.json_normalize(data, "attributes")
        #df1=pd.DataFrame(data)
        df2.to_csv('output.csv',mode='a', encoding='utf-8',index=False)

def write_csv(path):
    list_of_files = get_list_of_json_files()
    #print(list_of_files)

    for file in list_of_files:
        #path1=path+'\\'+file
        create_list_from_json(file)  # create the row to be added to csv for each file (json-file)

# Save the csv file in parquet format and partion on the basis of Status.
def convert_csvToParquet(csvfile):
        df=pd.read_csv(csvfile)
        table = pa.Table.from_pandas(df)
        pq.write_to_dataset(table,root_path='output1.parquet',partition_cols=['STATUS'])
        pandas_df=pd.read_parquet('output1.parquet')
        print(pandas_df)

# Get the total sales value of the cancelled orders
def query_total_sales_cancelled(parquetfile):
    pandas_df=pd.read_parquet('output1.parquet')
    sqlquery = "select sum(SALES) from pandas_df where STATUS='Cancelled'"
    print(ps.sqldf(sqlquery))

#  total sales value of the orders currently on hold for the year 2005
def query_totalsales_2005(parquetfile):
    pandas_df=pd.read_parquet('output1.parquet')
    sqlquery = "select sum(SALES) from pandas_df where STATUS='On Hold' and ORDERDATE like '%2005%'"
    print(ps.sqldf(sqlquery))

# Count of distinct products per product line
def query_count_distinct_productlines(parquetfile):
    pandas_df=pd.read_parquet('output1.parquet')
    sqlquery = "select PRODUCTLINE,count(*) from pandas_df group by PRODUCTLINE"
    print(ps.sqldf(sqlquery))

# Total sales variance for sales calculated at both sales price and MSRP
def query_variance(parquetfile):
    pandas_df=pd.read_parquet('output1.parquet')
    sqlquery_sales = "select sum(SALES)from pandas_df"
    sqlquery_expected = "select sum(MSRP*QUANTITYORDERED)from pandas_df"
    value_sales=ps.sqldf(sqlquery_sales)
    value_expected=ps.sqldf(sqlquery_expected)
    #total_variance=sqlquery_expected-sqlquery_sales
    print("Variance"+value_sales)

# Percentage change in sales YoY for classic cars
def query_PerChangeClassicCars(parquetfile):
    pandas_df=pd.read_parquet('output1.parquet')
    sqlquery_2004 = "select sum(SALES) from pandas_df where STATUS='Shipped' and ORDERDATE like '%2004%'"
    sqlquery_2005 = "select sum(SALES) from pandas_df where STATUS='Shipped' and ORDERDATE like '%2005%'"
    value_sales_2004=ps.sqldf(sqlquery_2004)
    value_expected_2005=ps.sqldf(sqlquery_2005)
    print(value_expected_2005)
    per_Change=((value_expected_2005-value_expected_2004)/value_expected_2004)*100
    total_variance=sqlquery_expected-sqlquery_sales
    print("PerChangeClassicCars"+total_variance)

# Percentage change in sales YoY for classic cars
def query_FilterProductLine(parquetfile):
    pandas_df=pd.read_parquet('output1.parquet')
    print(pandas_df[(pandas_df['PRODUCTLINE'] == 'Vintage Cars') | (pandas_df['PRODUCTLINE'] == 'Classic Cars') | (pandas_df['PRODUCTLINE'] == 'Motorcycles') | (pandas_df['PRODUCTLINE'] == 'Trucks and Buses') ])
    # print(pandas_df)

# Qunatity Based discounts
def query_QuantityBasedDiscount(parquetfile):
    pandas_df=pd.read_parquet('output1.parquet')
    pandas_df['QUANTITYORDERED'] = pd.to_numeric(pandas_df.QUANTITYORDERED, errors='coerce').fillna(0).astype(np.int64)
    pandas_df['SALES'] = pd.to_numeric(pandas_df.SALES, errors='coerce').fillna(0).astype(np.int64)
    pandas_df['DiscountedSale']=pandas_df['SALES']
    for index,item in pandas_df.iterrows():
        if item["QUANTITYORDERED"]<=30:
            pandas_df.iloc[index,7]= pandas_df.iloc[index,2]
        if item["QUANTITYORDERED"]>30 & item["QUANTITYORDERED"] <= 60:
            pandas_df.iloc[index]['DiscountedSale']= pandas_df.iloc[index]['SALES']*0.975
        if item["QUANTITYORDERED"] >60 & item["QUANTITYORDERED"] <=80:
            pandas_df.iloc[index]['DiscountedSale']= pandas_df.iloc[index]['SALES']*0.96
        if item["QUANTITYORDERED"] >80 & item["QUANTITYORDERED"] <=100:
            pandas_df.iloc[index]['DiscountedSale']= pandas_df.iloc[index]['SALES']*0.94
        if item["QUANTITYORDERED"] >100:
            pandas_df.iloc[index]['DiscountedSale']= pandas_df.iloc[index]['SALES']*0.90
    print(pandas_df)

def query_recalculateMSRP(parquetfile):
    pandas_df=pd.read_parquet('output1.parquet')
    pandas_df['QUANTITYORDERED'] = pd.to_numeric(pandas_df.QUANTITYORDERED, errors='coerce').fillna(0).astype(np.int64)
    pandas_df['SALES'] = pd.to_numeric(pandas_df.SALES, errors='coerce').fillna(0).astype(np.int64)
    pandas_df['RecalculatedSales']=pandas_df['SALES']/pandas_df['QUANTITYORDERED']
    print(pandas_df)

if __name__=="__main__":
    path=r'dataset'
    # Total sales value of the cancelled orders
    query_total_sales_cancelled('output1.parquet')

    # Total sales value of the orders currently on hold for the year 2005
    query_totalsales_2005('output1.parquet')

    # Cunt of distinct products per product line
    query_count_distinct_productlines('output1.parquet')

    # total sales variance for sales calculated at both sales price and MSRP
    query_variance('output1.parquet')

    #  percentage change in sales YoY for classic cars, for years 2004 and 2005 where the status is shipped
    query_PerChangeClassicCars('output1.parquet')

    # Filter the data from productline
    query_FilterProductLine('output1.parquet')

    # Quantity based sales column
    query_QuantityBasedDiscount('output1.parquet')

    # Calculated column by recalculating sales using MSRP.
    query_recalculateMSRP('output1.parquet')


import os
import configparser

root_path = os.path.join(os.path.dirname(
    os.path.realpath(__file__)), '..', '..')
config_path = os.path.join(root_path, 'config')
peers_path = os.path.join(config_path, 'peers.ini')

class CfgPeers:
    # for Alice/Bob and Charlie 1

    def read_config_peers(self, user):
        config = configparser.ConfigParser()
        config.read(peers_path)
        ip_address = config.get(user, 'ip_address')
        port_number = config.getint(user, 'port_number')
        return ip_address, port_number

    # for Tracker 2
    def read_config_peers_all(self):
        config_p = configparser.ConfigParser()
        config_p.read(peers_path)
        peers_info = {}
        for peer in config_p.sections():
            p = config_p.getint(peer, 'port_number')
            i = config_p.get(peer, 'ip_address')
            peers_info[peer] = (i, p)
        return peers_info


import torch
import numpy as np
import qpsolvers as qps
import quadprog
import itertools
from qpth.qp import QPFunction, QPSolvers

__author__ = "Xin Wang"
__credits__ = ["Xin Wang"]
__license__ = "MIT"
__version__ = "0.1"
__maintainer__ = "Xin Wang"
__email__ = "xin3.wang@intel.com"
__status__ = "Development"

def project_half_space(v, v0):
    """
    Simple projection to half space defined by the normal hyperplane of the primary gradient,
    done on device
    v: vector of dimension D, will be updated in place if projection is to be performed
    v0: vector of dimension D, the primary
    """
    v0 = torch.squeeze(v0[-1])
    # if torch.dot(v, v0) < 0.:
    #     print("cos(theta) = {:.4f}".format(torch.dot(v, v0)/torch.norm(v)/torch.norm(v0)))
    v += v0 * F.relu(-torch.dot(v, v0)) / torch.norm(v0)**2 # NOTE: inplace!

def project_convex_cone(v, v0, eps=1e-13, qp_solver='qpth'):
    """
    Projection to convex cone defined by normal hyperplanes of the primary vectors,
    done on cpu if using quadratic programming package, 'quadprog', 'cvxopt', or 'cvxpy'
    done on gpu if using 'qpth'
    v: vector of dimension D, will be updated in place if projection is to be performed
    v0: matrix of dimensions N * D, each of the N rows is a primary vector
    eps: tolerance of Gram matrix entries to trigger an QP optimization
    qp_solver: QP solver, 'qpth', 'quadprog', 'cvxopt', or 'cvxpy'
    """
    # dp = torch.matmul(v0, v)
    # if any(dp<0.):
    #     print("dot(v, v0) before: \33[91m{}\033[0m".format(', '.join(["%.4f" % x for x in torch.matmul(v0, v)])))

    N = v0.shape[0]

    if qp_solver=='qpth':
        Q = torch.matmul(v0, v0.t())
        if torch.any(Q.abs() > eps):
            p = torch.matmul(v, v0.t())
            _v = QPFunction(verbose=-1, solver=QPSolvers.CVXPY)(
                Q, p,
                -torch.eye(N, device=v0.device),
                torch.zeros(N, device=v0.device),
                torch.Tensor(), torch.Tensor()
            )
            v += torch.matmul(_v, v0).squeeze() # NOTE: inplace!
    else: # 'quadprog', 'cvxopt', or 'cvxpy'
        qp_solver = getattr(qps, qp_solver + '_solve_qp')
        Q = torch.matmul(v0, v0.t()).cpu().double().numpy()
        if np.any(np.abs(Q) > eps):
            p = torch.matmul(v, v0.t()).cpu().double().numpy()
            _v = qp_solver(Q, p, -np.eye(N), np.zeros(N))
            v += torch.matmul(v0.t(), torch.Tensor(_v).to(v0.device)) # NOTE: inplace!

    # if any(dp<0.):
    #     print("dot(v, v0) after: \33[92m{}\033[0m".format(', '.join(["%.4f" % x for x in torch.matmul(v0, v)])))


import ctypes
import json
import time
import requests
from api_key_file import api_key_f

def get_weather(location_key):
    api_key = api_key_f
    req2 = requests.get(
        f'http://dataservice.accuweather.com/currentconditions/v1/{location_key}?apikey={api_key}').json()
    return req2[0]["WeatherText"]

def main(city):
    api_key = api_key_f
    req = requests.get(
        f'http://dataservice.accuweather.com/locations/v1/cities/search?apikey={api_key}&q={city}').json()
    location_key = req[0]["Key"]
    req2 = requests.get(
        f'http://dataservice.accuweather.com/currentconditions/v1/{location_key}?apikey={api_key}').json()
    if req2:
        print(f'Weather right now: {req2[0]["WeatherText"]}')
        main2(location_key)
    else:
        print('Network Error...')

def main2(location_key):
    print('Here you can freely create your wallpaper based on weather outside c: \n')
    try:
        with open('data.txt', 'r') as file:
            data = file.read()
        print('If you want, you can add more wallpapers, just type "add"')
        print('Or print "start" to launch the program')

    except FileNotFoundError:
        with open('data.txt', 'w') as file:
            pass
        print('To start, you have to create your wallpapers \n')
        print(f'How to do that: \n'
              f'1. Type "add" \n'
              f'2. Type what weather this wallpaper will appear with \n'
              f'3. Paste absolute path to this wallpaper \n'
              f'4. Profit :3')

    w = input()
    if w == 'add':
        d = dict()
        w = input('Type what weather this wallpaper will appear with \n')
        print('Paste absolute path to this wallpaper \n')
        w2 = input()
        d[w] = w2
        with open('data.txt', 'w') as file:
            json.dump(d, file)

    elif w == 'start':
        print('Type "stop" anytime you want to stop the program')
        with open('data.txt', 'r') as file:
            data = json.load(file)
        while True:
            try:
                path = data[get_weather(location_key)]
                print(path)
                ctypes.windll.user32.SystemParametersInfoW(20, 0, path, 0)
            except KeyError:
                pass
            time.sleep(800)


'''
Dropout is easily implemented only during training by randomly selecting nodes
to be dropped-out with a given probability (e.g. 20%) each weight update cycle.
Only used during the training of a model and is not used when evaluating the
skill of the model.

Trained network is capable of better generalization and is less likely to
overfit the training data.
'''
import numpy
import pandas
from keras.constraints import maxnorm
from keras.layers import Dense, Dropout
from keras.models import Sequential
from keras.optimizers import SGD
from keras.wrappers.scikit_learn import KerasClassifier
from sklearn.model_selection import cross_val_score, StratifiedKFold
from sklearn.pipeline import Pipeline
from sklearn.preprocessing import LabelEncoder, StandardScaler

# Create and compile model
def create_model():
    model = Sequential()
    model.add(Dense(60, input_dim=60, init='normal', activation='relu',
                    W_constraint=maxnorm(3)))
    model.add(Dropout(0.2))
    model.add(Dense(30, init='normal', activation='relu',
                    W_constraint=maxnorm(3)))
    model.add(Dropout(0.2))
    model.add(Dense(1, init='normal', activation='sigmoid'))
    sgd = SGD(lr=0.1, momentum=0.9, decay=0.0, nesterov=False)
    model.compile(loss='binary_crossentropy', optimizer=sgd,
                  metrics=['accuracy'])
    return model

# Fix random seed for reproducibility
seed = 7
numpy.random.seed(seed)

# Load dataset
dataframe = pandas.read_csv("csv/sonar.csv", header=None)
dataset = dataframe.values
X = dataset[:, 0:60].astype(float)
Y = dataset[:, 60]

# Encode class values as integers
encoder = LabelEncoder()
encoder.fit(Y)
encoded_Y = encoder.transform(Y)

# Evaluate model with standardized dataset
estimators = []
estimators.append(('standardize', StandardScaler()))
estimators.append(('mlp', KerasClassifier(build_fn=create_model,
                  nb_epoch=300, batch_size=16, verbose=0)))

# Executes one or more models within a pass of the cross validation procedure
pipeline = Pipeline(estimators)

# 10-fold cross validation test
kfold = StratifiedKFold(n_splits=10, shuffle=True, random_state=seed)

# Evaluate a score by cross-validation
results = cross_val_score(pipeline, X, encoded_Y, cv=kfold)
print("Best: %.2f%% (%.2f%%)" %
      (results.mean()*100, results.std()*100))


# --------------------------- IMPORT MODULES ----------------------------------
import numpy as np
import matplotlib.pyplot as plt
import matplotlib.cm as cm
import sys
import operator
import random
import time
import math
import copy
import gc
import cProfile
from scipy.stats import norm, expon

from landscape_SETUP import *

#*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*- SAVE GRIDS *-*-*-*-*-*-*-*-*-*-*-*-*-*-*-

def savegrids(sand_heights_grid, veg_grid, apparent_veg_type_grid, age_grid, windspeed_grid, w_soil_moisture_grid, saving_loc, t):
    #----------------------- Function description -----------------------------
    #Save data to grids

    #---------------------------------- Start ---------------------------------
    saving_grid_sand[(saving_loc*Nr):(saving_loc*Nr)+Nr, 0:Nc] = sand_heights_grid
    saving_grid_veg[(saving_loc*Nr):(saving_loc*Nr)+Nr, 0:Nc] = veg_grid
    saving_grid_apparent_veg_type[(saving_loc*Nr):(saving_loc*Nr)+Nr, 0:Nc] = apparent_veg_type_grid
    saving_grid_age[(saving_loc*Nr):(saving_loc*Nr)+Nr, 0:Nc] = age_grid
    if (t+1) % wind_event_frequency == 0:
        saving_grid_wind[(saving_loc*Nr):(saving_loc*Nr)+Nr, 0:Nc] = windspeed_grid
        saving_grid_moisture[(saving_loc*Nr):(saving_loc*Nr)+Nr, 0:Nc] = w_soil_moisture_grid[((2*n_shells)+1):(Nrw+1), ((2*n_shells)+1):(Ncw+1)]
    saving_loc += 1

    return (saving_grid_sand, saving_grid_veg, saving_grid_apparent_veg_type, saving_grid_age, saving_grid_wind, saving_grid_moisture, saving_loc)
#*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-

#*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*- START ARRAYS *-*-*-*-*-*-*-*-*-*-*-*-*-*-

def startarrays():
    #----------------------- Function description -----------------------------
    #Define initial arrays to fill out with data

    #---------------------------------- Start ---------------------------------
    rainfall_days = np.zeros(model_iterations)  #Defines which days it has rained or not (1=rain, 0=not)
    total_sand_vol = np.zeros(model_iterations) #Total volumne of sand being transported at each iteration
    total_aval_vol = np.zeros(model_iterations) #Volume of sand avalanching at each iteration
    total_veg_pop = np.zeros(veg_iterations) #Total veg cover in the grid at each iteration
    average_age_table = np.zeros((veg_iterations, 3)) #Average age of each veg type at each iteration
    veg_proportions = np.zeros((veg_iterations, 3)) #Proportions of each veg type at each iteration
    exposed_wall_proportions = np.zeros(model_iterations) #% of walls that are fully exposed (i.e. aren't covered in sand) in the grid at each iteration
    differences_grid = np.zeros((Nrw+2, Ncw+2)) #Grid to allow differences in erosion/deposition to be calculated
    avalanching_grid = np.zeros((Nrw, Ncw)) #Grid to form basis for avalanching grid calculations
    windspeed_grid = np.zeros((Nr, Nc)) #Grid to form basis for windspeed calculations
    interaction_field = np.zeros((Nr, Nc)) #Grid to form basis for neighbourhood interactions between plants

    return (rainfall_days, total_sand_vol, total_aval_vol, total_veg_pop, average_age_table, veg_proportions, exposed_wall_proportions, differences_grid, avalanching_grid, windspeed_grid, interaction_field)
#*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-

#*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*- START GRIDS *-*-*-*-*-*-*-*-*-*-*-*-*-*-*

def startgrids():
    #----------------------- Function description -----------------------------
    #Define initial grid of sand heights and vegetation

    #-------------------------- Function setup --------------------------------
    walls_presence_grid = np.zeros((Nr, Nc)) #Grid of 1's and 0's to show where wall is present
    grid = np.zeros((Nr, Nc)) #Grid of 1's and 0's to turn neighbourhood contributions on or off
    drought_grid = np.zeros((Nr, Nc)) #Grid of zeros to add consecutive number of droughts each plant experiences
    veg_type_grid = np.zeros((Nr, Nc)) #Grid to fill with plant types - 0 (no plant), 1 (grass), 2 (shrub), 3 (tree)
    veg_grid = np.zeros((Nr, Nc)) #Grid to fill with plant heights
    cum_growth_grid = np.zeros((Nr, Nc)) #Grid to fill with cumulated plant growth
    actual_biomass_grid = np.zeros((Nr, Nc)) #Grid to fill with the actual biomass that the cumulated growth represents
    chance3 = np.random.random((Nr, Nc)) #Grid of random numbers for allocating plant type randomly to cells

    #---------------------------------- Start ---------------------------------
    #------------------------------- Sand heights -----------------------------
    #Define initial grid of sand heights
    if start_grid_sand_mode == 'random': #Create grid of random sand heights
        sand_heights_grid = np.zeros((Nr, Nc))
        for r in range(Nr):
            for c in range(Nc):
                sand_heights_grid[r, c] = np.random.uniform(start_grid_sand_lower, start_grid_sand_upper) #Fill with a random number from 0 to desired max height
    elif start_grid_sand_mode == 'uniform': #Create grid of uniform heights as specified
        sand_heights_grid = (np.ones((Nr, Nc)))*start_grid_sand_upper
    elif start_grid_sand_mode == 'circle': #Create grid of bare cells with a circle of sand at the centre
        n_r = Nr-1; n_c = Nc-1; circle_centre_r = np.round(Nr/2); circle_centre_c = np.round(Nc/2)
        y, x = np.ogrid[-circle_centre_r:n_r-circle_centre_r, -circle_centre_c:n_c-circle_centre_c]; mask = x**2 + y**2 <= sand_circle_radius**2
        sand_heights_grid = np.ones((Nr, Nc))*0; sand_heights_grid[mask] = sand_circle_height
    else:
        print('WARNING: No start_grid_sand_mode defined!')

    #------------------------------- Wall heights -----------------------------
    #Define initial grid of walls
    if start_grid_wall_mode == 'none': #No walls in domain
        walls_grid = np.zeros((Nr, Nc))
    elif start_grid_wall_mode == 'circle': #Circular filled wall at centre of grid
        n_r = Nr-1; n_c = Nc-1; circle_centre_r = np.round(Nr/2); circle_centre_c = np.round(Nc/2)
        y, x = np.ogrid[-circle_centre_r:n_r-circle_centre_r, -circle_centre_c:n_c-circle_centre_c]; mask = x**2 + y**2 <= wall_circle_or_square_radius_or_length**2
        walls_grid = np.ones((Nr, Nc))*0; walls_grid[mask] = wall_circle_or_square_height
    elif start_grid_wall_mode == 'square': #Square filled wall at centre of grid
        square_centre_r = np.round(Nr/2); square_centre_c = np.round(Nc/2)
        r_start = square_centre_r - wall_circle_or_square_radius_or_length; r_end = r_start + (2*wall_circle_or_square_radius_or_length)
        c_start = square_centre_c - wall_circle_or_square_radius_or_length; c_end = c_start + (2*wall_circle_or_square_radius_or_length)
        walls_grid = np.zeros((Nr, Nc)); walls_grid[r_start:r_end, c_start:c_end] = wall_circle_or_square_height
    elif start_grid_wall_mode == 'manual': #Manually defined walls using indexing
        walls_grid = np.zeros((Nr, Nc)); walls_grid[10:15, 10:15] = 3; walls_grid[20:25, 20:25] = 2.5
    else:
        print('WARNING: No start_grid_wall_mode defined!')

    walls_presence_grid[np.where(walls_grid > 0)] = 1 #Fill grid with 1's where walls are present

    #---------------------------------- Ages ----------------------------------
    #Create an initial plant population of ages
    if start_grid_age_mode == 'random': #Random ages up to maximum random age (in months)
        age_grid_random = np.random.random((Nr, Nc))
        age_grid = np.zeros((Nr, Nc))
        age_grid[np.where(age_grid_random <= veg_distrib)] = 1.
        age_grid[np.where(age_grid_random > veg_distrib)] = 0.
        age_grid = (age_grid*(np.round(np.random.random((Nr, Nc))*start_grid_age))) + veg_update_freq_equivalent #Add 1 at the end to make every plant cell at least 1 month old (they won't have any height at 1 month
    elif start_grid_age_mode == 'uniform': #Every vegetated cell has same age
        age_grid_random = np.random.random((Nr, Nc))
        age_grid = np.zeros((Nr, Nc))
        age_grid[np.where(age_grid_random <= veg_distrib)] = 1.
        age_grid[np.where(age_grid_random > veg_distrib)] = 0.
        age_grid = (age_grid*start_grid_age) + veg_update_freq_equivalent #Add veg_update_freq_equivalent at the end to make every plant cell at least 1 x veg_update_freq_equivalent
    else:
        print('WARNING: No start_grid_age_mode defined!')

    #Update 'grid' for neighbourhood interactions and the occupation grid
    grid[np.where(age_grid > veg_update_freq_equivalent)] = 1
    veg_occupation_grid = copy.copy(grid) #Grid of 1's and 0's to signify where vegetation exists

    #---------------------------- Veg types and heights -----------------------
    #Define initial grids of veg type, veg height, growth units and biomass according to age
    if (grass_proportion + shrub_proportion + tree_proportion) != 1:
        print('WARNING: Grass/shrub/tree proportions do not sum to 1!')

    for R in range(Nr):
        for C in range(Nc):
            cell_age = age_grid[R, C]
            if cell_age > veg_update_freq_equivalent: #If there is a plant in the cell
                if chance3[R, C] <= grass_proportion: #GRASS
                    veg_type_grid[R, C] = 1 #'1' means the vegetation type is a grass
                    starting_biomass =  max_biomass_grass/(1+(math.exp(-biomass_exp_factor_grass*(cell_age - biomass_midpoint_growth_grass)))) #Starting biomass is the theoretically perfect biomass for that age (taken from veg_update routine)
                    actual_biomass_grid[R, C] = starting_biomass
                    veg_grid[R, C] = (starting_biomass/max_biomass_grass)*max_height_grass #Simple proportional conversion from biomass units to veg height
                    cum_growth_grid[R, C] = cell_age #Age and cumulative growth units are theoretically the same to start
                elif chance3[R, C] <= (grass_proportion + shrub_proportion): #SHRUB
                    veg_type_grid[R, C] = 2 #'2' means the vegetation type is a shrub
                    starting_biomass =  max_biomass_shrub/(1+(math.exp(-biomass_exp_factor_shrub*(cell_age - biomass_midpoint_growth_shrub)))) #Starting biomass is the theoretically perfect biomass for that age (taken from veg_update routine)
                    actual_biomass_grid[R, C] = starting_biomass
                    veg_grid[R, C] = (starting_biomass/max_biomass_shrub)*max_height_shrub #Simple proportional conversion from biomass units to veg height
                    cum_growth_grid[R, C] = cell_age #Age and cumulative growth units are theoretically the same to start
                elif chance3[R, C] <= (grass_proportion + shrub_proportion + tree_proportion): #TREE
                    veg_type_grid[R, C] = 3 #'3' means the vegetation type is a tree
                    starting_biomass =  max_biomass_tree/(1+(math.exp(-biomass_exp_factor_tree*(cell_age - biomass_midpoint_growth_tree)))) #Starting biomass is the theoretically perfect biomass for that age (taken from veg_update routine)
                    actual_biomass_grid[R, C] = starting_biomass
                    veg_grid[R, C] = (starting_biomass/max_biomass_tree)*max_height_tree #Simple proportional conversion from biomass units to veg height
                    cum_growth_grid[R, C] = cell_age #Age and cumulative growth units are theoretically the same to start
            else: #If there is no plant present in cell
                if chance3[R, C] <= grass_proportion: #GRASS
                    veg_type_grid[R, C] = 1
                elif chance3[R, C] <= (grass_proportion + shrub_proportion): #SHRUB
                    veg_type_grid[R, C] = 2
                elif chance3[R, C] <= (grass_proportion + shrub_proportion + tree_proportion): #TREE
                    veg_type_grid[R, C] = 3
                cum_growth_grid[R, C] = veg_update_freq_equivalent #Minimum growth unit
                actual_biomass_grid[R, C] = veg_update_freq_equivalent #Minimum growth unit
                veg_grid[R, C] = 0. #No veg height

    #Remove plants from walls, as they can grow in the same place as a wall!
    age_grid[np.where(walls_grid > 0)] = 0;
    veg_grid[np.where(walls_grid > 0)] = 0;

    return (sand_heights_grid, veg_grid, veg_type_grid, age_grid, cum_growth_grid, actual_biomass_grid, veg_occupation_grid, drought_grid, grid, walls_grid, walls_presence_grid)
#*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-

#*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-* START GRIDS 2 *-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-

def startgrids_2(veg_grid, veg_type_grid):
    #----------------------- Function description -----------------------------
    #Define initial grids of vegetation properties: veg type, porosity, trunk height
    #'veg_grid' is grid of veg heights (Nr x Nc)
    #'veg_type_grid' is grid of veg types: grass, shrub or tree (Nr x Nc)

    #-------------------------- Function setup --------------------------------
    porosity_grid = np.zeros((Nr, Nc))
    trunks_grid = np.zeros((Nr, Nc))
    w_soil_moisture_grid = np.zeros((Nrw+2, Ncw+2))

    #---------------------------------- Start ---------------------------------
    #------------------------- Veg porosities ---------------------------------
    porosity_grid[np.where(veg_grid > 0)] = 1.
    if start_grid_porosity_mode == 'random': #Allocate random porosities to veg
        porosity_grid = porosity_grid*(np.random.uniform(min_random_porosity, max_random_porosity, (Nr, Nc)))
    elif start_grid_porosity_mode == 'uniform': #Every plant has same porosity
        porosity_grid = porosity_grid*start_grid_porosity
    else:
        print('WARNING: No start_grid_porosity_mode defined!')

    #---------------------- Trunk heights for trees ---------------------------
    trunks_grid[np.where(veg_type_grid == 3)] = 1.
    if start_grid_trunks_mode == 'random': #Allocate random trunk proportions to veg
        trunks_grid = trunks_grid*veg_grid*(np.random.uniform(min_trunk_proportion, max_trunk_proportion, (Nr, Nc))) #Make the trunk height a proportion of the tree height (observing minimum and maximum proportions the trunk can be)
    elif start_grid_trunks_mode == 'uniform': #Every tree cell has same proportion of total height that is trunk
        trunks_grid = trunks_grid*veg_grid*start_grid_trunks
    else:
        print('WARNING: No start_grid_trunks_mode defined!')

    return (porosity_grid, trunks_grid, w_soil_moisture_grid)
#*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-

#*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-* START GRIDS 3 *-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-
def startgrids_3(sand_heights_grid, veg_grid, veg_type_grid):
    #----------------------- Function description -----------------------------
    #Define initial grids for things like sediment balance calculations
    #'veg_grid' is grid of veg heights (Nr x Nc)
    #'veg_type_grid' is grid of veg types: grass, shrub or tree (Nr x Nc)

    #---------------------------------- Start ---------------------------------
    initial_sand_heights_grid = copy.copy(sand_heights_grid)
    rooting_heights_grid = copy.copy(sand_heights_grid) #Rooting heights assumed to be at the initial sand surface
    initial_veg_grid = copy.copy(veg_grid)
    initial_apparent_veg_type_grid = copy.copy(veg_type_grid); initial_apparent_veg_type_grid[np.where(initial_veg_grid == 0)] = 0 #It's APPARENT because all cells are attributed a potential veg type, but those that have zero height need to be seen as zero
    apparent_veg_type_grid = veg_type_grid #Assign this ahead of routine in case veg isn't updated during the run
    sed_balance_moisture_grid = np.zeros((Nrw+2, Ncw+2)) #Just blank sediment balance to begin with

    return (initial_sand_heights_grid, rooting_heights_grid, initial_veg_grid, initial_apparent_veg_type_grid, apparent_veg_type_grid, sed_balance_moisture_grid)
#*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-

#*-*-*-*-*-*-*-*-*-*-*-*- START GRIDS MANUAL -*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*
def startgrids_manual():
    #----------------------- Function description -----------------------------
    #Define initial grids manually (using specific matrix indexing, or by loading existing files)

    #---------------------------------- Start ---------------------------------
    #Define main grids manually
    sand_heights_grid = np.ones((Nr, Nc))*1; #sand_heights_grid[20:40, 20:40] = 0.4
    walls_grid = np.zeros((Nr, Nc)); walls_grid[10:15, 10:15] = 2.2; walls_grid[20:25, 20:25] = 4 #Grid of where solid walls are located
    veg_grid = np.ones((Nr, Nc))*1; veg_grid[np.where(walls_grid > 0)] = 0; #veg_grid[10:40, 10:40] = 4
    age_grid = np.ones((Nr, Nc))*300; age_grid[np.where(walls_grid > 0)] = 0; #age_grid[10:40, 10:40] = 300
    veg_type_grid = np.ones((Nr, Nc)); #veg_type_grid[10:40, 10:40] = 2 #Beware - can't have any zeros in veg_type_grid (a veg type has to be ascribed)

    #Define main grids from previous files
    #sand_heights_grid = np.loadtxt('Final_sand_grid_barchans1', delimiter=',')
    #veg_grid = np.loadtxt('/Users/jeromemayaud/Documents/University/Oxford/DPhil/RESULTS/Modelling/Python/MODEL/MODEL_v6/Veg_grid', delimiter=',')

    #Define secondary grids depending on the main grids
    walls_presence_grid = np.zeros((Nr, Nc)); walls_presence_grid[np.where(walls_grid > 0)] = 1 #Fill grid with 1's where walls are present
    cum_growth_grid = age_grid #Make the cumulated biomass same as initial age - i.e. plants are perfectly healthy
    actual_biomass_grid = age_grid #Ditto
    grid = np.zeros((Nr, Nc)); grid[np.where(age_grid > veg_update_freq_equivalent)] = 1 #For neighbourhood interactions
    veg_occupation_grid = copy.copy(grid) #Grid of 1's and 0's to signify where vegetation exists
    drought_grid = np.zeros((Nr, Nc)) #Grid of zeros to add consecutive number of droughts each plant experiences

    return (sand_heights_grid, veg_grid, veg_type_grid, age_grid, cum_growth_grid, actual_biomass_grid, veg_occupation_grid, drought_grid, grid, walls_grid, walls_presence_grid)
#*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-