from ansible.module_utils.basic import AnsibleModule
from PyPowerStore.utils.exception import PowerStoreException
from ansible.module_utils import dellemc_ansible_utils as utils
import logging
from uuid import UUID

__metaclass__ = type
ANSIBLE_METADATA = {'metadata_version': '1.0',
                    'status': ['preview'],
                    'supported_by': 'community'
                    }

DOCUMENTATION = r'''
---
module: dellemc_powerstore_volumegroup
version_added: '2.6'
short_description: Manage volume groups on PowerStore Storage System
description:
- Managing volume group on PowerStore Storage System includes
  creating new volume group, adding volumes to volume
  group, removing volumes from volume group, renaming volume group,
  modifying volume group and deleting volume group.
author:
- Akash Shendge (akash.shendge@dell.com)
- Arindam Datta (arindam.datta@dell.com)
extends_documentation_fragment:
  - dellemc.dellemc_powerstore
options:
  vg_name:
    description:
    - The name of the volume group.
    required: False
  vg_id:
    description:
    - The id of the volume group.
    - It can be used only for Modify, Add/Remove or Delete operation.
    required: False
  volumes:
    description:
    - This is a list of volumes.
    - Either the volume ID or name must be provided for adding/removing
      existing volumes from volume group.
    - If volumes are given, then vol_state should also be specified.
  vol_state:
    description:
    - String variable , describes the state of volumes inside VG
    - If volume are given, then vol_state should also be specified.
    choices: [present-in-group , absent-in-group]
  new_vg_name:
    description:
     - The new name of the volume group.
  description:
    description:
    - Description about the volume group.
  protection_policy:
    description:
    - String variable, represents Protection policy id or name
      used for volume group.
    - Specifying an empty string or "" removes the existing
      protection policy from Volume Group.
    required: false
  is_write_order_consistent:
    description:
    - A boolean flag to indicate whether snapshot sets of the volume group will be write-order consistent.
    - If this parameter is not specified, the array by default sets it to true.
    required: false
  state:
    description:
    - Define whether the volume group should exist or not.
    choices: [absent, present]
    required: true
Notes:
- vol_state is mandatory if volumes are provided.
- A protection policy can be specified either for an volume group, or
  for the individual volumes inside the volume group.
- A volume can be a member of at most one volume group.
- Specifying "protection_policy" as empty string or "" removes the existing
  protection policy from a Volume Group.

'''

EXAMPLES = r'''
- name: Create volume group without protection policy
  dellemc_powerstore_volumegroup:
    array_ip: "{{array_ip}}"
    verifycert: "{{verifycert}}"
    user: "{{user}}"
    password: "{{password}}"
    vg_name: "{{vg_name}}"
    description: "This volume group is for ansible"
    state: "present"

 - name: Get details of volume group
   dellemc_powerstore_volumegroup:
     array_ip: "{{array_ip}}"
     verifycert: "{{verifycert}}"
     user: "{{user}}"
     password: "{{password}}"
     vg_name: "{{vg_name}}"
     state: "present"

 - name: Add volumes to volume group
   dellemc_powerstore_volumegroup:
     array_ip: "{{array_ip}}"
     verifycert: "{{verifycert}}"
     user: "{{user}}"
     password: "{{password}}"
     vg_name: "{{vg_name}}"
     state: "present"
     volumes:
       - "7f879569-676c-4749-a06f-c2c30e09b295"
       - "68e4dad5-5de5-4644-a98f-6d4fb916e169"
       - "Ansible_Testing"
     vol_state: "present-in-group"

- name: Remove volumes from volume group
  dellemc_powerstore_volumegroup:
    array_ip: "{{array_ip}}"
    verifycert: "{{verifycert}}"
    user: "{{user}}"
    password: "{{password}}"
    vg_name: "{{vg_name}}"
    state: "present"
    volumes:
      - "7f879569-676c-4749-a06f-c2c30e09b295"
      - "Ansible_Testing"
    vol_state: "absent-in-group"

- name: Rename volume group and change is_write_order_consistent flag
  dellemc_powerstore_volumegroup:
    array_ip: "{{array_ip}}"
    verifycert: "{{verifycert}}"
    user: "{{user}}"
    password: "{{password}}"
    vg_name: "{{vg_name}}"
    new_vg_name: "{{new_vg_name}}"
    is_write_order_consistent: False
    state: "present"

- name: Get details of volume group by ID
  dellemc_powerstore_volumegroup:
    array_ip: "{{array_ip}}"
    verifycert: "{{verifycert}}"
    user: "{{user}}"
    password: "{{password}}"
    vg_id: "{{vg_id}}"
    state: "present"

- name: Delete volume group
  dellemc_powerstore_volumegroup:
    array_ip: "{{array_ip}}"
    verifycert: "{{verifycert}}"
    user: "{{user}}"
    password: "{{password}}"
    name: "{{new_vg_name}}"
    state: "absent"

'''

RETURN = r'''
'''

LOG = utils.get_logger('dellemc_powerstore_volumegroup',
                       log_devel=logging.INFO)

py4ps_sdk = utils.has_pyu4ps_sdk()
HAS_PY4PS = py4ps_sdk['HAS_Py4PS']
IMPORT_ERROR = py4ps_sdk['Error_message']

py4ps_version = utils.py4ps_version_check()
IS_SUPPORTED_PY4PS_VERSION = py4ps_version['supported_version']
VERSION_ERROR = py4ps_version['unsupported_version_message']

# Application type
APPLICATION_TYPE = 'Ansible/1.0'

class PowerStoreVolumeGroup(object):
    """Class with Volume Group operations"""

    def __init__(self):
        """Define all the parameters required by this module"""
        self.module_params = utils.get_powerstore_management_host_parameters()
        self.module_params.update(get_powerstore_volume_group_parameters())

        # initialize the Ansible module
        self.module = AnsibleModule(
            argument_spec=self.module_params,
            supports_check_mode=True
        )

        LOG.info('HAS_PY4PS = {0} , IMPORT_ERROR = {1}'.format(
                 HAS_PY4PS, IMPORT_ERROR))
        if HAS_PY4PS is False:
            self.module.fail_json(msg=IMPORT_ERROR)
        LOG.info('IS_SUPPORTED_PY4PS_VERSION = {0} , '
                 'VERSION_ERROR = {1}'.format(
                  IS_SUPPORTED_PY4PS_VERSION, VERSION_ERROR))
        if IS_SUPPORTED_PY4PS_VERSION is False:
            self.module.fail_json(msg=VERSION_ERROR)

        self.conn = utils.get_powerstore_connection(self.module.params,
          application_type=APPLICATION_TYPE)
        self.provisioning = self.conn.provisioning
        LOG.info('Got Py4ps instance for provisioning on PowerStore {0}'.
                 format(self.conn))
        self.protection = self.conn.protection
        LOG.info('Got Py4ps instance for protection on PowerStore {0}'.
                 format(self.protection))

    def get_volume_group_details(self, vg_id=None, name=None):
        """Get volume group details"""
        try:
            LOG.info("Getting VG Details {0} {1}".format(vg_id, name))

            if vg_id:
                resp = self.provisioning.get_volume_group_details(vg_id)
                LOG.info("Successfully Got VG with id {0}".format(vg_id))
                if len(resp) > 0:
                    return resp
                else:
                    return None

            if name:
                resp = self.provisioning.get_volume_group_by_name(name)
                if resp and len(resp) > 0:
                    id = resp[0]['id']
                    vg_details = self.provisioning.\
                        get_volume_group_details(id)
                    LOG.info("Successfully Got VG with name {0}".format(name))
                    return vg_details

            return None

        except Exception as e:

            id_or_name = vg_id if vg_id else name
            errormsg = "Failed to get volume group {0} with error {1}". \
                format(id_or_name, str(e))
            LOG.error(errormsg)

    def delete_volume_group(self, volume_group_id):
        """Delete volume group"""
        try:
            self.provisioning.delete_volume_group(volume_group_id)
            return True

        except PowerStoreException as pe:
            errormsg = "Failed to delete volume group {0} with error {1}". \
                format(volume_group_id, str(pe))
            LOG.error(errormsg)
            self.module.fail_json(msg=errormsg)

        except Exception as e:
            errormsg = "Failed to delete volume group {0} with error {1}".\
                format(volume_group_id, str(e))
            LOG.error(errormsg)
            self.module.fail_json(msg=errormsg)

    def get_volume_id_by_name(self, vol_name):
        """Get the details of a volume by name"""

        try:
            resp = self.provisioning.get_volume_by_name(vol_name)
            if len(resp) > 0:
                return resp[0]['id']
            else:
                return None

        except Exception as e:
            msg = "Failed to get the volume with name {0} with " \
                  "error {1}".format(vol_name, str(e))
            LOG.error(msg)

    def get_volume_details_by_id(self, volume_id):
        """Get the details of a volume by name"""

        try:
            LOG.info("getting volume with id {0}".format(volume_id))
            resp = self.provisioning.get_volume_details(volume_id)
            if len(resp) > 0:
                LOG.info("got volume with id {0} is {1}".format(volume_id, resp))
                return resp['id']
            else:
                return None

        except Exception as e:
            msg = "Failed to get the volume with id {0} with " \
                  "error {1}".format(volume_id, str(e))
            LOG.error(msg)

    def remove_volumes_from_volume_group(self, vg_id, vol_list):
        """Remove volumes from volume group"""

        vol_group_details = self.get_volume_group_details(vg_id=vg_id)
        existing_volumes_in_vg = vol_group_details['volumes']
        LOG.debug("Existing Volumes: {0}".format(existing_volumes_in_vg))

        existing_vol_ids = []
        for vol in existing_volumes_in_vg:
            if vol:
                existing_vol_ids.append(vol['id'])

        LOG.debug("Existing Volume IDs {0}".format(existing_vol_ids))

        ids_to_remove = []

        vol_name_list = []
        vol_id_list = []

        for each_vol in vol_list:
            if each_vol:
                identifier_type = self.name_or_id(each_vol)
                if identifier_type == "ID" and not (each_vol in vol_id_list):
                    vol_id_list.append(each_vol)
                elif identifier_type == "NAME" and not (each_vol in vol_name_list):
                    vol_name_list.append(each_vol)

        """remove by name"""
        for vol in vol_name_list:
            id = self.get_volume_id_by_name(vol)
            if id and (id in existing_vol_ids):
                if id not in ids_to_remove:
                    ids_to_remove.append(id)
            else:
                msg = "Unable to remove volume name {0} since is not " \
                      "present in volume group {1}".format(vol, vg_id)
                LOG.warn(msg)
                self.module.fail_json(msg=msg)

        """remove by id"""
        for vol in vol_id_list:
            if vol in existing_vol_ids:
                if vol not in ids_to_remove:
                    ids_to_remove.append(vol)
                else:
                    msg = "Unable to remove volume id {0} since is not " \
                          "present in volume group {1}".format(vol, vg_id)
                    LOG.warn(msg)
                    self.module.fail_json(msg=msg)

        LOG.debug("Volume IDs to Remove {0}".format(ids_to_remove))

        if len(ids_to_remove) == 0:
            return False

        try:
            self.provisioning.remove_members_from_volume_group(
                vg_id, ids_to_remove)

            return True

        except PowerStoreException as pe:
            errormsg = "Remove existing volume(s) from volume group {0} " \
                       "failed with error {1}".format(vg_id, str(pe))
            LOG.error(errormsg)
            self.module.fail_json(msg=errormsg)

        except Exception as e:
            errormsg = "Remove existing volume(s) from volume group {0} " \
                       "failed with error {1}".format(vg_id, str(e))
            LOG.error(errormsg)
            self.module.fail_json(msg=errormsg)

    def add_volumes_to_volume_group(self,vg_id, vol_list):
        """adds volumes to volume group"""

        vol_group_details = self.get_volume_group_details(vg_id=vg_id)
        existing_volumes_in_vg = vol_group_details['volumes']
        LOG.debug("Existing Volumes: {0}".format(existing_volumes_in_vg))
        existing_vol_ids = []
        for vol in existing_volumes_in_vg:
            if vol:
                existing_vol_ids.append(vol['id'])

        LOG.debug("Existing Volume IDs {0}".format(existing_vol_ids))

        ids_to_add = []
        vol_name_list = []
        vol_id_list = []

        for each_vol in vol_list:
            if each_vol:
                identifier_type = self.name_or_id(each_vol)
                if identifier_type == "ID" and not (each_vol in vol_id_list):
                    vol_id_list.append(each_vol)
                elif identifier_type == "NAME" and not (each_vol in vol_name_list):
                    vol_name_list.append(each_vol)

        """add volume by name"""
        for vol in vol_name_list:
            id = self.get_volume_id_by_name(vol)
            if id and (id not in existing_vol_ids):
                if id not in ids_to_add:
                    ids_to_add.append(id)
            else:
                msg = "Unable to add volume name {0}, either it doesn't" \
                      " exist or already in volume group ".format(vol)
                LOG.warn(msg)
                self.module.fail_json(msg=msg)

        """add volume by id"""
        for vol in vol_id_list:
            """verifying if volume id exists in array"""
            vol_by_id = self.get_volume_details_by_id(volume_id=vol)

            if vol_by_id not in existing_vol_ids:
                if vol_by_id not in ids_to_add:
                    ids_to_add.append(vol_by_id)
            else:
                msg = "Unable to add volume id {0}, either it doesn't" \
                      " exist or already in volume group ".format(vol)
                LOG.warn(msg)
                self.module.fail_json(msg=msg)

        LOG.debug("Volume IDs to add {0}".format(ids_to_add))

        if len(ids_to_add) == 0:
            return False

        try:
            self.provisioning.add_members_to_volume_group(
                vg_id, ids_to_add)

            return True
        except PowerStoreException as pe:
            errormsg = "Add existing volumes to volume group {0} " \
                       "failed with error {1}".format(vg_id, str(pe))
            LOG.error(errormsg)
            self.module.fail_json(msg=errormsg)

        except Exception as e:
            errormsg = "Add existing volumes to volume group {0} " \
                       "failed with error {1}".format(vg_id, str(e))
            LOG.error(errormsg)
            self.module.fail_json(msg=errormsg)

    def modify_volume_group(self, vg_id, vg_name, description,
                            is_write_order_consistent,
                            protection_policy_id):
        """Modify volume group"""
        try:
            LOG.info("Modifying volume group: {0}".format(
                vg_id))

            self.provisioning.modify_volume_group(
                vg_id, vg_name, description, is_write_order_consistent,
                protection_policy_id)

            LOG.info("Successfully modified volume group: {0}".format(
                vg_id))

            return True

        except PowerStoreException as pe:
            errormsg = "Modify volume group {0} failed with error {1}". \
                format(vg_id, str(pe))
            LOG.error(errormsg)
            self.module.fail_json(msg=errormsg)

        except Exception as e:
            errormsg = "Modify volume group {0} failed with error {1}".\
                format(vg_id, str(e))
            LOG.error(errormsg)
            self.module.fail_json(msg=errormsg)

    def is_volume_group_modified(self, volume_group,protection_policy):
        """Check if the desired volume group state is different from existing
        volume group"""
        modified = False

        if(('name' in volume_group and self.module.params['new_vg_name']
            is not None) and (volume_group['name'].lower() !=
                              self.module.params['new_vg_name'].lower())):
            modified = True
        elif((volume_group['description'] is not None and self.module.params[
            'description'] is not None and volume_group['description'].lower()
              != self.module.params['description'].lower()) or
             (volume_group['description'] is None and
              self.module.params['description'] is not None and
              self.module.params['description'].lower() != 'none')):
            modified = True
        elif(('protection_policy_id' in volume_group and
              protection_policy is not None and
              volume_group['protection_policy_id'] !=
              protection_policy) or
                ('protection_policy_id' not in volume_group and
                 protection_policy is not None and
                 protection_policy.lower() != 'none')):
            modified = True
        elif('is_write_order_consistent' in volume_group and
                self.module.params['is_write_order_consistent'] is not None
                and volume_group['is_write_order_consistent'] !=
                self.module.params['is_write_order_consistent']):
            modified = True

        return modified

    def create_volume_group(self, vg_name,
                            description,
                            protection_policy_id,
                            is_write_order_consistent):
        """Create a volume group"""
        try:
            LOG.info('Creating empty volume group {0} '.format(vg_name))
            resp = self.provisioning.create_volume_group(
                vg_name, description,
                protection_policy_id=protection_policy_id,
                is_write_order_consistent=is_write_order_consistent)
            return True, resp

        except PowerStoreException as pe:
            errormsg = "Failed to create volume group {0} with error {1}". \
                format(vg_name, str(pe))
            LOG.error(errormsg)
            self.module.fail_json(msg=errormsg)

        except Exception as e:
            errormsg = "Failed to create volume group {0} with error {1}".\
                format(vg_name, str(e))
            LOG.error(errormsg)
            self.module.fail_json(msg=errormsg)

    def get_protection_policy_id_by_name(self, name):
        """Get protection policy by name"""
        try:
            LOG.info('Getting the details of protection policy '
                     '{0}'.format(name))
            resp = self.protection.get_protection_policy_by_name(name)
            if None is resp or len(resp) <= 0:
                msg = 'No protection policy present with name {0}'.format(
                    name)
                LOG.error(msg)
                self.module.fail_json(msg=msg)
            else:
                LOG.info('Successfully got the protection policy '
                         'name {0}'.format(name))
                return resp[0]['id']

        except Exception as e:
            msg = 'Get details of protection policy name: {0} failed' \
                  ' with error : {1} '.format(name, str(e))
            LOG.error(msg)
            self.module.fail_json(msg=msg)

    def get_protection_policy_details_by_id(self, policy_id):
        """Get protection policy details by id"""

        try:
            LOG.info('Getting the details of protection policy by id'
                     ' {0}'.format(policy_id))
            resp = self.protection.get_protection_policy_details(
                policy_id=policy_id)
            if resp and len(resp) > 0:
                LOG.info('Successfully got the details of '
                         'protection policy id {0}'.format(policy_id))

                return resp['id']
            else:
                msg = 'No protection policy present with id {0}'.format(
                    policy_id)
                LOG.error(msg)
                self.module.fail_json(msg=msg)

        except Exception as e:
            msg = 'Get details of protection policy id: {0} failed' \
                  ' with error {1}'.format(policy_id, str(e))
            LOG.error(msg)
            self.module.fail_json(msg=msg)

    def name_or_id(self, val):
        """Determines if the string is a name or id"""
        try:
            UUID(str(val))
            LOG.info('{0} is a ID'.format(val))
            return "ID"
        except ValueError:
            LOG.info('{0} is a NAME'.format(val))
            return "NAME"

    def perform_module_operation(self):
        """
        Perform different actions on volume group based on user parameter
        chosen in playbook
        """

        vg_id = self.module.params['vg_id']
        state = self.module.params['state']
        vg_name = self.module.params['vg_name']
        volumes = self.module.params['volumes']
        vol_state = self.module.params['vol_state']
        new_vg_name = self.module.params['new_vg_name']
        description = self.module.params['description']
        protection_policy = self.module.params['protection_policy']
        is_write_order_consistent = self.module.params[
            'is_write_order_consistent']

        volume_group = None

        if not vg_name and not vg_id:
            self.module.fail_json(msg='Specify volume group name or '
                                      'volume group id')
        elif vg_name and vg_id:
            self.module.fail_json(msg='Specify volume group name or '
                                      'volume group id, not both')

        volume_group = self.get_volume_group_details(vg_id=vg_id, name=vg_name)
        LOG.debug('volume_group details: {0}'.format(volume_group))

        if protection_policy:
            prot_pol_identifier_type = self.name_or_id(protection_policy)
            if prot_pol_identifier_type == "ID":
                protection_policy = self.get_protection_policy_details_by_id(
                    protection_policy)
            if prot_pol_identifier_type == "NAME":
                protection_policy = self.get_protection_policy_id_by_name(
                    protection_policy)

        modified = False

        if volume_group:
            modified = self.is_volume_group_modified(volume_group,protection_policy)
            LOG.debug('Modified Flag: {0}'.format(modified))
        else:
            if not vg_name:
                self.module.fail_json(msg="vg_name is required to "
                                          "create a Volume Group")
            if new_vg_name:
                self.module.fail_json(msg="Invalid argument, "
                                          "new_vg_name is not required")
        if volumes and not vol_state:
            msg="vol_state must also be specified along with Volumes"
            LOG.error(msg)
            self.module.fail_json(msg=msg)

        if vg_id is None and volume_group:
            vg_id = volume_group['id']
        if vg_name is None and volume_group:
            vg_name = volume_group['name']

        result = dict(
            changed=False,
            create_vg='',
            modify_vg='',
            add_vols_to_vg='',
            remove_vols_from_vg='',
            delete_vg='',
            volume_group_details='',
        )

        if state == 'present' and not volume_group:
            LOG.info('Creating volume group {0}'.format(vg_name))
            result['create_vg'], resp = self.\
                create_volume_group(vg_name, description,
                                    protection_policy,
                                    is_write_order_consistent)
            result['volume_group_details'] = resp
            volume_group = self.get_volume_group_details(vg_id=resp['id'])
            vg_id = volume_group['id']
        elif state == 'absent' and volume_group:
            LOG.info('Deleting volume group {0}'.format(vg_id))
            result['delete_vg'] = self.delete_volume_group(vg_id)

        if state == 'present' and vol_state == 'present-in-group' and \
                volume_group and volumes:
            result['add_vols_to_vg'] = self.add_volumes_to_volume_group(
                vg_id, volumes)
        elif state == 'present' and vol_state == 'absent-in-group' and \
                volume_group and volumes:
            LOG.info('Remove existing volume(s) from volume group {0}'.
                     format(vg_id))
            result['remove_vols_from_vg'] = self.\
                remove_volumes_from_volume_group(vg_id, volumes)

        if state == 'present' and volume_group and modified:
            LOG.info("From Modify : {0}".format(protection_policy))
            result['modify_vg'] = self.modify_volume_group(
                 vg_id, new_vg_name, description, is_write_order_consistent,
                 protection_policy)

        if state == 'present' and volume_group:
            updated_vg = self.get_volume_group_details(vg_id=vg_id)
            result['volume_group_details'] = updated_vg

        if result['create_vg'] or result['modify_vg'] or result[
            'add_vols_to_vg'] or result['remove_vols_from_vg'] or \
                result['delete_vg']:
            result['changed'] = True

        self.module.exit_json(**result)

def get_powerstore_volume_group_parameters():
    return dict(
        vg_name=dict(required=False, type='str'),
        vg_id=dict(required=False, type='str'),
        new_vg_name=dict(required=False, type='str'),
        volumes=dict(required=False, type='list'),
        vol_state=dict(required=False, choices=['absent-in-group',
                                                'present-in-group'],
                       type='str'),
        state=dict(required=True, choices=['present', 'absent'], type='str'),
        description=dict(required=False, type='str'),
        is_write_order_consistent=dict(required=False, type='bool'),
        protection_policy=dict(required=False, type='str')
    )

def main():
    """Create PowerStore volume group object and perform action on it
        based on user input from playbook"""
    obj = PowerStoreVolumeGroup()
    obj.perform_module_operation()

if __name__ == '__main__':
    main()


# do some analysis of variance
import sys
import csv
from tabulate import tabulate
from scipy import stats

ANOVA_HEADERS = ['src', 'DF', 'SS', 'MS', 'F', 'p']

def sum_x(data):
    total = 0
    for row in data:
        total += sum(row)
    return total

def sum_xsq(data):
    total = 0
    for row in data:
        for item in row:
            total += item**2
    return total

def prep_data(file, reps):
    reader = csv.reader(file)
    init_data = []
    for line in reader:
        init_data.append([float(item) for item in line])

    n_rows = len(init_data)
    n_cols = len(init_data[0])
    n = n_cols * n_rows

    gridsum_data = []
    for i in range(0, n_rows, reps):
        summed_line = []
        for j in range(n_cols):
            block_sum = 0
            for k in range(i, i+reps, 1):
                block_sum += init_data[k][j]
            summed_line.append(block_sum)
        gridsum_data.append(summed_line)

    col_sums = [0 for _ in range(n_cols)]
    row_sums = []
    for row in gridsum_data:
        row_sums.append(sum(row))
        for i in range(len(row)):
            col_sums[i] += row[i]

    grand_total = sum(col_sums)

    return init_data, gridsum_data, col_sums, row_sums, grand_total, n, n_cols, n_rows

def pretty_anova_tbl(anova, sources):
    anova_tbl = []
    for src in sources:
        line = [src, ]
        for attr in ANOVA_HEADERS[1:]:
            if attr in anova[src]:
                line.append(anova[src][attr])
            else:
                line.append(None)
        anova_tbl.append(line)

    print(tabulate(anova_tbl, headers=ANOVA_HEADERS))

def simple_anova(file, reps):

    data, _, treat_sums, _, grand_total, n, treat_n, obs_per_treat = prep_data(file, reps)

    anova = {}; sources = ['treats', 'err', 'total']
    for item in sources:
        anova[item] = {}

    # DF's
    anova['treats']['DF'] = treat_n-1
    anova['total']['DF'] = n-1
    anova['err']['DF'] = anova['total']['DF'] - anova['treats']['DF']

    # SS's
    anova['total']['SS'] = sum_xsq(data) - ((sum_x(data)**2)/n)
    ssTreats = 0
    for i in range(treat_n):
        ssTreats += (treat_sums[i]**2)/obs_per_treat
    ssTreats -= (grand_total**2)/n
    anova['treats']['SS'] = ssTreats
    anova['err']['SS'] = anova['total']['SS'] - anova['treats']['SS']

    # MS's
    anova['treats']['MS'] = anova['treats']['SS']/anova['treats']['DF']
    anova['err']['MS'] = anova['err']['SS']/anova['err']['DF']

    # F
    anova['treats']['F'] = anova['treats']['MS']/anova['err']['MS']

    # p
    anova['treats']['p'] = stats.f(anova['treats']['DF'], anova['err']['DF']).sf(anova['treats']['F'])

    pretty_anova_tbl(anova, sources)

def blocked_anova(file, reps):

    data, _, treat_sums, block_sums, grand_total, n, n_treats, obs_per_treat = prep_data(file, reps)
    n_blocks = obs_per_treat//reps
    obs_per_block = n // n_blocks

    anova = {}; sources = ['treats', 'blocks', 'err', 'total']
    for item in sources:
        anova[item] = {}

    # DF's
    anova['total']['DF'] = n-1
    anova['treats']['DF'] = n_treats-1
    anova['blocks']['DF'] = n_blocks-1
    anova['err']['DF'] = anova['total']['DF'] - anova['treats']['DF'] - anova['blocks']['DF']

    # SS's
    anova['total']['SS'] = sum_xsq(data) - ((sum_x(data) ** 2) / n)
    ssTreats = 0
    for i in range(n_treats):
        ssTreats += (treat_sums[i] ** 2) / obs_per_treat
    ssTreats -= (grand_total ** 2) / n
    anova['treats']['SS'] = ssTreats
    ssBlocks = 0
    for i in range(n_blocks):
        ssBlocks += (block_sums[i]**2) / obs_per_block
    ssBlocks -= (grand_total**2) / n
    anova['blocks']['SS'] = ssBlocks
    anova['err']['SS'] = anova['total']['SS'] - anova['treats']['SS'] - anova['blocks']['SS']

    # MS's
    anova['treats']['MS'] = anova['treats']['SS'] / anova['treats']['DF']
    anova['blocks']['MS'] = anova['blocks']['SS'] / anova['blocks']['DF']
    anova['err']['MS'] = anova['err']['SS'] / anova['err']['DF']

    # F
    anova['treats']['F'] = anova['treats']['MS'] / anova['err']['MS']
    anova['blocks']['F'] = anova['blocks']['MS'] / anova['err']['MS']

    # p
    anova['treats']['p'] = stats.f(anova['treats']['DF'], anova['err']['DF']).sf(anova['treats']['F'])
    anova['blocks']['p'] = stats.f(anova['blocks']['DF'], anova['err']['DF']).sf(anova['blocks']['F'])

    pretty_anova_tbl(anova, sources)

def twoway_anova(file, reps):
    data, gridsum_data, fac1_sums, fac2_sums, grand_total, n, n_fac1s, obs_per_fac1 = prep_data(file, reps)
    n_fac2s = obs_per_fac1 // reps
    obs_per_fac2 = n // n_fac2s
    obs_per_square = obs_per_fac1 // n_fac2s

    anova = {}; sources = ['faccol', 'facrow', 'inter', 'err', 'total']
    for item in sources:
        anova[item] = {}

    # DF's
    anova['total']['DF'] = n - 1
    anova['faccol']['DF'] = n_fac1s-1
    anova['facrow']['DF'] = n_fac2s-1
    anova['inter']['DF'] = anova['faccol']['DF'] * anova['facrow']['DF']
    anova['err']['DF'] = anova['total']['DF'] - anova['faccol']['DF'] - anova['facrow']['DF'] - anova['inter']['DF']

    # SS's
    anova['total']['SS'] = sum_xsq(data) - ((sum_x(data) ** 2) / n)
    SSFac1 = 0
    for sum in fac1_sums:
        SSFac1 += (sum**2)/obs_per_fac1
    SSFac1 -= grand_total**2 / n
    anova['faccol']['SS'] = SSFac1
    SSFac2 = 0
    for sum in fac2_sums:
        SSFac2 += (sum**2)/obs_per_fac2
    SSFac2 -= grand_total**2 / n
    anova['facrow']['SS'] = SSFac2
    SSInter = 0
    for i in range(n_fac2s):  # rows
        for j in range(n_fac1s):  # cols
            SSInter += gridsum_data[i][j]**2 / obs_per_square
    SSInter -= grand_total**2 / n
    SSInter -= SSFac1 + SSFac2
    anova['inter']['SS'] = SSInter
    anova['err']['SS'] = anova['total']['SS'] - anova['faccol']['SS'] - anova['facrow']['SS'] - anova['inter']['SS']

    # MS's
    anova['faccol']['MS'] = anova['faccol']['SS'] / anova['faccol']['DF']
    anova['facrow']['MS'] = anova['facrow']['SS'] / anova['facrow']['DF']
    anova['inter']['MS'] = anova['inter']['SS'] / anova['inter']['DF']
    anova['err']['MS'] = anova['err']['SS'] / anova['err']['DF']

    # F
    anova['faccol']['F'] = anova['faccol']['MS'] / anova['err']['MS']
    anova['facrow']['F'] = anova['facrow']['MS'] / anova['err']['MS']
    anova['inter']['F'] = anova['inter']['MS'] / anova['err']['MS']

    # p
    anova['faccol']['p'] = stats.f(anova['faccol']['DF'], anova['err']['DF']).sf(anova['faccol']['F'])
    anova['facrow']['p'] = stats.f(anova['facrow']['DF'], anova['err']['DF']).sf(anova['facrow']['F'])
    anova['inter']['p'] = stats.f(anova['inter']['DF'], anova['err']['DF']).sf(anova['inter']['F'])

    pretty_anova_tbl(anova, sources)

if __name__ == '__main__':
    if len(sys.argv) < 4:
        print('Usage: python3', sys.argv[0], '[simple|blocked|twoway]', '[csv_file]', '[reps]')
        exit()

    method = sys.argv[1]
    reps = int(sys.argv[3])
    with open(sys.argv[2]) as file:
        if method == 'simple':
            simple_anova(file, reps)
        elif method == 'blocked':
            blocked_anova(file, reps)
        elif method == 'twoway':
            twoway_anova(file, reps)
        else:
            print('Unknown method:', method)
            print('Allowed methods: simple, blocked, twoway')


import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns; sns.set()
import matplotlib.pyplot as plt
import csv
import numpy as np
import matplotlib.mlab

def plotDecisionRegionQDA(sampleMean, CV_, training, data):
    all_res = []
    minX, maxX, minY, maxY = 0., 8., 0., 8.

    # create one-dimensional arrays for x and y
    x = np.linspace(minX, maxX, 100, endpoint=True)
    y = np.linspace(minY, maxY, 100, endpoint=True)
    cnt = 0
    for i in range(len(x) - 1):
        for k in range(len(y) - 1):
            classIndex = 0
            logLikelihood = float("inf")
            for j in range(len(data.species.unique())):
                Nk = training.groupby('species').size()[data.species.unique()[j]]
                newLikelihood = logLikelihoodCalc([x[i],y[k]], sampleMean[j], CV_[j], Nk)
                if (newLikelihood < logLikelihood):
                    logLikelihood = newLikelihood
                    classIndex = j
            test = [x[i],y[k], data.species.unique()[classIndex]]
            test = pd.DataFrame([test], columns=['pet_length', 'pet_width', 'species'])
            cnt  =cnt + 1
            all_res.append(test)
    plotPairwiseScatterPlot(pd.concat(all_res))

def plotDecisionRegionLDA(sampleMean, CV_, training, data):
    all_res = []
    minX, maxX, minY, maxY = 0., 8., 0., 8.

    # create one-dimensional arrays for x and y
    x = np.linspace(minX, maxX, 100, endpoint=True)
    y = np.linspace(minY, maxY, 100, endpoint=True)
    cnt = 0
    for i in range(len(x) - 1):
        for k in range(len(y) - 1):
            classIndex = 0
            logLikelihood = float("inf")
            for j in range(len(data.species.unique())):
                Nk = training.groupby('species').size()[data.species.unique()[j]]
                newLikelihood = logLikelihoodCalc([x[i],y[k]], sampleMean[j], CV_, Nk)
                if (newLikelihood < logLikelihood):
                    logLikelihood = newLikelihood
                    classIndex = j
            test = [x[i],y[k], data.species.unique()[classIndex]]
            test = pd.DataFrame([test], columns=['pet_length', 'pet_width', 'species'])
            cnt  =cnt + 1
            all_res.append(test)
    plotPairwiseScatterPlot(pd.concat(all_res))

def plotPairwiseScatterPlot(dataframe):
    sns.scatterplot(x="pet_length", y="pet_width", hue="species",
                      data=dataframe)

def plotDecisionRegion(C, mu ):
    x = np.arange(-5, 10.0, 0.005)
    y = np.arange(-5, 10.0, 0.005)
    X, Y = np.meshgrid(x, y)
    Z = matplotlib.mlab.bivariate_normal(X, Y, C[0, 0], C[1, 1], mu[0], mu[1], (C[0,1]*C[1,0]))
    plt.contour(X, Y, Z, levels = 10)

def Mahalonobis(x,C):
    return x.dot(np.linalg.inv(C)).dot(np.transpose(x))

#-log likelihood
def logLikelihoodCalc(data,mu, C, Nk):
    return (1/2 * np.log(np.linalg.det(C)) + np.log(Mahalonobis(data-mu,C)) - np.log(Nk))

def generateSampleMean(data):
    mean = [ data["pet_length"].mean(),data["pet_width"].mean()]
    return mean
def generateClassSampleMeans(data):
    classMeans = []
    classes = data.species.unique()
    for species in classes:
        classFrame = data.loc[data['species'] == species]
        classMeans.append(generateSampleMean(classFrame))
    return np.asarray(classMeans)

def generateClassCovarianceMatrix(data, mu):
    classCVs = []
    classes = data.species.unique()
    j=0
    mu = np.asarray(mu)
    for species in classes:
        classFrame = data.loc[data['species'] == species]
        classCVs.append(generatePooledCovariance(classFrame,mu[j]))
        j = j+1

    return np.asarray(classCVs)

def generatePooledCovariance(data, mean):
    data = pd.DataFrame(data)
    C  = np.ndarray(shape=(2,2), dtype=float, order='F')
    mean = np.asarray(mean)
    data.reset_index(inplace=True, drop = True)
    for i in range(len(data.index)):
        x_ = [data["pet_length"][i],data["pet_width"][i]]
        for j in  range(len(data.species.unique())):
            if data.species.unique()[j] == data["species"][i]:
                C = C + 1/len(data.index)*np.outer((x_-mean[j]),np.transpose(x_-mean[j]))
    return C

def LOOCVQDA(data, mode):
    LOOCV_Error = 0.0;
    data = data.reset_index()
    for i in range(len(data.index)):
        training = data.copy(deep=True)
        training.drop(training.index[i], inplace=True)
        test = data.iloc[i]
        classIndex = 0
        logLikelihood = float("inf")
        sampleMean = generateClassSampleMeans(training)
        CV_ = generateClassCovarianceMatrix(training, sampleMean)
        for j in range(CV_.shape[0]):
            if mode > 0:
                CV_[j]= np.diag(np.diag(CV_[j]))
                if mode > 1:
                    CV_[j] = (np.trace(CV_[j]) / CV_[j].shape[1]) * \
                          np.identity(CV_[j].shape[1])
        for j in range(len(data.species.unique())):
            Nk = training.groupby('species').size()[data.species.unique()[j]]
            newLikelihood = logLikelihoodCalc([test['pet_length'], test['pet_width']], sampleMean[j], CV_[j], Nk)
            if (newLikelihood < logLikelihood):
                logLikelihood = newLikelihood
                classIndex = j
        if (i == len(data.index) - 1):
            #plotPairwiseScatterPlot(training)
            plotDecisionRegionQDA(sampleMean, CV_, training, data)
        if data.species.unique()[classIndex] != test["species"]:
            LOOCV_Error = LOOCV_Error + 1 / len(data.index);

    return LOOCV_Error

def LOOCVLDA(data, mode):
    LOOCV_Error = 0.0;
    data = data.reset_index()
    for i in range(len(data.index)):
        training = data.copy(deep=True)
        training.drop(training.index[i], inplace=True)
        test = data.iloc[i]
        classIndex = 0
        logLikelihood = float("inf")
        sampleMean = generateClassSampleMeans(training)
        CV_ = generatePooledCovariance(training,sampleMean)

        if mode > 0:
            CV_ = np.diag(np.diag(CV_))
            if mode > 1:
                CV_ =( np.trace(CV_) / CV_.shape[0]) * \
                     np.identity(CV_.shape[0])
        for j in range(len(data.species.unique())):
            #if(i == len(data.index) - 1):
                #plotDecisionRegion(CV_, sampleMean[j])
            Nk =training.groupby('species').size()[data.species.unique()[j]]
            newLikelihood = logLikelihoodCalc([test['pet_length'],test['pet_width']],sampleMean[j], CV_, Nk)
            if( newLikelihood < logLikelihood):
                logLikelihood = newLikelihood
                classIndex = j
        if (i == len(data.index) - 1):
            plotPairwiseScatterPlot(training)
            plotDecisionRegionLDA(sampleMean, CV_, training, data)
        if data.species.unique()[classIndex] != test["species"]:
            LOOCV_Error = LOOCV_Error + 1/len(data.index);

    return LOOCV_Error

with open("iris.data.txt", 'rt') as csvfile:
     iris = csv.reader(csvfile, delimiter=',', quotechar='|')
     IrisReduced = pd.DataFrame(list(iris))
IrisReduced.columns = ['sep_length','sep_width','pet_length','pet_width','species']
IrisReduced.drop(['sep_length'], axis = 1, inplace = True)
IrisReduced.drop(['sep_width'], axis = 1, inplace = True)
IrisReduced[['pet_length','pet_width']] = IrisReduced[['pet_length','pet_width']].astype(float)

#LDA mode: 0-general, 1-independent, 2- isotropic
print("Error for LDA: general case")
print(LOOCVLDA(IrisReduced, 1))
#print("Error for LDA: independent case")
#print(LOOCVLDA(IrisReduced, 1))
#print("Error for LDA: isotropic case")
#print(LOOCVLDA(IrisReduced, 2))

#QDA mode: 0-general, 1-independent, 2- isotropic
#print("Error for QDA: general case")
#print(LOOCVQDA(IrisReduced, 0))
#print("Error for QDA: in independent case")
#print(LOOCVQDA(IrisReduced, 1))
#print("Error for QDA: isotropic case")
#print(LOOCVQDA(IrisReduced, 2))
plt.show()


import config
import telebot
import requests
from telebot import types

bot = telebot.TeleBot(config.token)

# Декодую json
response = requests.get(config.url).json()

@bot.message_handler(commands=['start', 'help'])
def send_welcome(message):
    markup = types.ReplyKeyboardMarkup(resize_keyboard=True, row_width=2)
    itembtn1 = types.KeyboardButton('USD')
    itembtn2 = types.KeyboardButton('EUR')

    markup.add(itembtn1, itembtn2,)
    msg = bot.send_message(message.chat.id,
                    "Дізнатися курс ПриватБанка ", reply_markup=markup)
    bot.register_next_step_handler(msg, process_coin_step)

def process_coin_step(message):
    try:
       markup = types.ReplyKeyboardRemove(selective=False)

       for coin in response:
           if (message.text == coin['ccy']):
              bot.send_message(message.chat.id, printCoin(coin['buy'], coin['sale']),
                               reply_markup=markup, parse_mode="Markdown")

    except Exception as e:
       bot.reply_to(message, 'ooops!')

def printCoin(buy, sale):
    '''Вивід курсу користувачу'''
    return "💰 *Курс купівлі:* " + str(buy) + "\n💰 *Курс продажу:* " + str(sale)

# Enable saving next step handlers to file "./.handlers-saves/step.save".
# Delay=2 means that after any change in next step handlers (e.g. calling register_next_step_handler())
# saving will hapen after delay 2 seconds.
bot.enable_save_next_step_handlers(delay=2)

# Load next_step_handlers from save file (default "./.handlers-saves/step.save")
# WARNING It will work only if enable_save_next_step_handlers was called!
bot.load_next_step_handlers()

if __name__ == '__main__':
    bot.polling(none_stop=True)

Telegram bot (курс П.Б.)/telegram bot P_B_.py


import apache_beam as beam

with beam.Pipeline() as pipeline:
  plants = (
      pipeline
      | 'Garden plants' >> beam.Create([
          ('🍓', 'Strawberry'),
          ('🥕', 'Carrot'),
          ('🍆', 'Eggplant'),
          ('🍅', 'Tomato'),
          ('🥔', 'Potato'),
      ])
      | 'Values' >> beam.Values()
      | beam.Map(print))


# coding= utf-8
import json
import mitmproxy.http

# 存在中文乱码情况
class GetJson:

    def response(self, flow: mitmproxy.http.HTTPFlow):
        if "https://stock.xueqiu.com/v5/stock/batch/quote.json?_t" in flow.request.url and "x=" in flow.request.url:
            # 数据的模拟
            data = json.loads(flow.response.text)
            with open("./data/quote.json", "w", encoding="UTF-8") as f:
                json.dump(data, f,indent=2,ensure_ascii=False)
            print("加载入文件完成...")

addons = [
    GetJson()
]

if __name__ == '__main__':
    from mitmproxy.tools.main import mitmdump

    mitmdump(['-p', '8080', "-s", __file__])


# -*- coding: utf-8 -*-
"""
Created on Wed Nov 29 21:18:31 2017

@author: jagan
"""

#Import Library
import matplotlib.pyplot as plt
import numpy as np

#Main Function
def main():
    N= 5
    width = 0.35

    #Data for the bar chart
    men_score=(75,70,98,62,55)
    wom_score=(65,77,98,85,63)

    ind = np.arange(N)

    p1=plt.bar(ind,men_score,width)
    p2=plt.bar(ind,wom_score,width)

    #label axis
    plt.title("Matplot Class - Bar Chart") # Adds title to graph
    plt.xlabel("Number")
    plt.ylabel("Score")
    plt.legend((p1[0],p2[0]),('Men','Women'))
    plt.show()

if __name__=="__main__":
    main()


import logging.handlers
import re
import sys
import tqdm

class Utilities(object):
	"""
	Utility functions.
	"""
	HEADERS = {
		'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/56.0.2924.87 Safari/537.36',
		'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
		'Accept-Language': 'en-US,en;q=0.5',
		'Accept-Encoding': 'gzip, deflate',
		'Connection': 'close'
	}

	URL_REGEX = re.compile(r'(https?:\/\/[\d\w.:-]+)', re.I)
	FORM_REGEX = re.compile(r'method[\'\"= ]{1,4}post[\'\"]?', re.I)
	OPEN_REGEX = re.compile(r'name[\'\"= ]{1,4}notesview[\'\"]?', re.I)
	ACCOUNT_REGEX = re.compile(r'/([a-f0-9]{32}/[a-f0-9]{32})', re.I)
	USER_FIELD_REGEX = re.compile(r'user.+', re.I)
	REDIRECT_FIELD_REGEX = re.compile(r'redirect.+', re.I)
	NAMES_REGEX = re.compile(r'name[\'\"= ]{1,4}notesview[\'\"]?', re.I)
	WEBADMIN_REGEX = re.compile(r'<title>.*administration</title>', re.I)
	RESTRICTED_REGEX = re.compile(r'(notes exception|not authorized)', re.I)
	VERSION_REGEX = re.compile(r'(?:version|domino administrator|domino|release)[=":\s]{0,4}([\d.]+)(?:\s|\")?', re.I)
	LINUX_USER_REGEX = re.compile(r'([a-z0-9-_].+):(.+)', re.I)
	WINDOWS_USER_REGEX = re.compile(r'(.+)\\(.+)', re.I)
	PATH_REGEX = re.compile(r'DataDirectory\s*=\s*\'(.+)\';', re.I)

	def set_logging(self):
		"""
		Configure the basic logging environment for the application.
		"""
		logger = logging.getLogger('DomiOwned')
		logger.setLevel(logging.DEBUG)
		custom_format = CustomLoggingFormatter()
		handler = logging.StreamHandler()
		handler.setFormatter(custom_format)
		logger.addHandler(handler)

		return logger

	def parse_credentials(self, value):
		"""
		Handle credentials if value is None.
		"""
		return '' if value is None else value

	def check_url(self, url):
		"""
		Check for valid base URL.
		"""
		if self.URL_REGEX.search(url):
			return self.URL_REGEX.search(url).group(1)
		else:
			self.logger.error('Invalid URL provided')
			sys.exit()

	def setup_progress(self, total):
		"""
		Setup progress bar.
		"""
		progress_bar = tqdm.tqdm(
			total=total,
			desc="Progress",
			smoothing=0.5,
			bar_format='{desc}{percentage:3.0f}%|{bar}|({n_fmt}/{total_fmt})|{elapsed} '
		)

		return progress_bar

class CustomLoggingFormatter(logging.Formatter):
	"""
	Custom logging formatter.
	"""
	DEBUG_FORMAT = "\033[1m\033[34m[*]\033[0m %(msg)s"
	INFO_FORMAT = "\033[1m\033[32m[+]\033[0m %(msg)s"
	WARN_FORMAT = "\033[1m\033[33m[!]\033[0m %(msg)s"
	ERROR_FORMAT = "\033[1m\033[31m[-]\033[0m %(msg)s"

	def __init__(self):
		super().__init__(fmt="%(levelno)d: %(msg)s", datefmt=None, style='%')

	def format(self, record):
		orig_format = self._style._fmt

		if record.levelno == logging.DEBUG:
			self._style._fmt = CustomLoggingFormatter.DEBUG_FORMAT
		elif record.levelno == logging.INFO:
			self._style._fmt = CustomLoggingFormatter.INFO_FORMAT
		elif record.levelno == logging.WARN:
			self._style._fmt = CustomLoggingFormatter.WARN_FORMAT
		elif record.levelno == logging.ERROR:
			self._style._fmt = CustomLoggingFormatter.ERROR_FORMAT

		result = logging.Formatter.format(self, record)
		self._style._fmt = orig_format

		return result

class Banner(object):
	"""
	Domi-Owned visual banner.
	"""
	SHOW = """
         __________   __________ __________
        |          |\|          |          |\\
        |  *    *  |||  *  *  * |        * ||
        |  *    *  |||          |     *    ||
        |  *    *  |||  *  *  * |  *       ||
        |__________|||__________|__________||
        |          || `---------------------`
        |  *    *  ||
        |          ||
        |  *    *  ||
        |__________||
         `----------`

             IBM/Lotus Domino OWNage
"""


# _*_ coding: utf-8 _*_

from flask import render_template, redirect, url_for, flash, request
from ..models import User, News, Weather
from . import main
from .. import HWMonitor
from flask_login import login_required, current_user, login_user
from sqlalchemy import desc
from random import sample

# 首页
@main.route('/', methods=['GET', 'POST'])
def index():
    # 如果当前未登录，登入测试帐号
    if not current_user.is_authenticated:
        user = User.query.filter_by(id=100).first()
        login_user(user)

    # 幻灯片部分
    news_count = News.query.filter_by(news_agency='BBC').order_by(desc(News.id)).first().id  # 获取数据库中最后一条新闻的id
    slider = []
    if news_count >= 20:
        selected_news = sample(range(news_count - 10, news_count), 5)  # 随机获得5条新闻的id
        for news in selected_news:
            slider.append(News.query.filter_by(id=news).first())

    # 分页和新闻部分
    page = request.args.get('page', 1, type=int)
    pagination = News.query.order_by(desc(News.id)).paginate(page, per_page=12, error_out=False)
    articles = pagination.items

    # 天气部分
    weather = Weather.query.filter_by(city="Tianjin").order_by(desc(Weather.id)).first()

    # 系统状态部分
    hw_status = HWMonitor()

    return render_template("index.html",
                           articles=articles,
                           weather=weather,
                           pagination=pagination,
                           slider=slider,
                           hw=hw_status)

# 抓取新闻接口 默认50条
@main.route('/fetchnews', methods=['GET', 'POST'])
@login_required
def fetch_news():
    News.fetch_news(50)
    return redirect(url_for('.index'))


import datetime

# Types of fields.
CHAR = "C"
NUMERAL = "N"
DATE = "D"
LOGICAL = "L"

# System encoding which is used to convert field names between bytes and string.
SYSTEM_ENCODING = "ascii"

# Reference data

ENCODINGS = {
    # got from dbf description [dbfspec]
    # id      name      description
    0x00: ("ascii", "ASCII"),  # internal use
    0x01: ("cp437", "DOS USA"),
    0x02: ("cp850", "DOS Multilingual"),
    0x03: ("cp1252", "Windows ANSI"),
    0x04: ("mac_roman", "Standard Macintosh"),  # NOT SHURE
    0x64: ("cp852", "EE MS-DOS"),
    0x65: ("cp866", "Russian MS-DOS"),
    0x66: ("cp865", "Nordic MS-DOS"),
    0x67: ("cp861", "Icelandic MS-DOS"),
    0x6A: ("cp737", "Greek MS-DOS (437G)"),
    0x6B: ("cp857", "Turkish MS-DOS"),
    0x96: ("mac_cyrillic", "Russian Macintosh"),
    0x97: ("mac_latin2", "Eastern Europe Macintosh"),  # NOT SHURE
    0x98: ("mac_greek", "Greek Macinstosh"),
    0xC8: ("cp1250", "Windows EE"),
    0xC9: ("cp1251", "Russian Windows"),
    0xCA: ("cp1254", "Turkish Windows"),
    0xCB: ("cp1253", "Greek Windows"),
    ## These encodings are not supported by Python but a part of DBF spec.
    # 0x68:    ('cp895', 'Kamenicky (Czech) MS-DOS'),
    # 0x69:    ('cp790', 'Mazovia (Polish) MS-DOS'),
}

REVERSE_ENCODINGS = dict(
    [(value[0], (code, value[1])) for code, value in ENCODINGS.items()]
)

SIGNATURES = {
    0x02: "FoxBase",
    0x03: "dBASE III",
    0x04: "dBASE IV",
    0x05: "dBASE V",
    0x30: "Visual FoxPro",
    0x31: "Visual FoxPro with AutoIncrement field",
    0x43: "dBASE IV with SQL table and memo file",
    0x7B: "dBASE IV with memo file",
    0x83: "dBASE III with memo file",
    0x8B: "dBASE IV with memo file",
    0x8E: "dBASE IV with SQL table",
    0xB3: ".dbv and .dbt memo (Flagship)",
    0xCB: "dBASE IV with SQL table and memo file",
    0xE5: "Clipper SIX driver with SMT memo field",
    0xF5: "FoxPro with memo field",
    0xFB: "FoxPro",
}

SUPPORTED_SIGNATURES = (0x03, 0x04, 0x05)

# <   -- little endian
# B   -- version number (signature)
# 3B  -- last update (YY, MM, DD)
# L   -- number of records
# H   -- length of header
# H   -- length of each record
# 17x -- pad (2B -- reserved,
#              B -- incomplete transaction,
#              B -- encryption flag,
#             4B -- free record thread (reserved for LAN)
#             8B -- reserved for multiuser dBASE
#              B -- MDX flag)
# B   -- language driver
# 2x  -- pad (2B -- reserved)
HEADER_FORMAT = "<B3BLHH17xB2x"

# <   -- little endian
# 11s -- field name in ASCII (terminated by 0x00)
# c   -- field type (ASCII)
# 4x  -- field data address ( 2B -- address in memory (for dBASE)
#                          OR 4B -- offset of field from
#                                   beginning of record (for FoxPro)
# B   -- field length
# B   -- decimal count
# 14x -- pad (2B -- reserved for multi-user dBASE,
#              B -- work area id ()
#             2B -- reserved for multi-user dBASE,
#              B -- flag for SET FIELDS
#             7B -- reserved
#              B -- index field flag)
# B   -- language driver
# 2x  -- pad (2B -- reserved)
FIELD_DESCRIPTION_FORMAT = "<11sc4xBB14x"

# Common functions

def dbf2date(dbf_str):
    """
    Converts date from dbf-date to datetime.date

    Args:
        `dbf_str`:
            string in format YYYYMMDD
    """
    if dbf_str is None or not dbf_str.isdigit() or len(dbf_str) != 8:
        result = None
    else:
        result = datetime.date(int(dbf_str[:4]), int(dbf_str[4:6]), int(dbf_str[6:8]))
    return result

def date2dbf(dt):
    """
    Converts date from datetime.date to dbf-date (string in format YYYYMMDD)

    Args:
        `dt`:
            datetime.date instance
    """
    if not isinstance(dt, datetime.date):
        raise TypeError("Espects datetime.date instead of %s" % type(dt))
    return b"%04d%02d%02d" % (dt.year, dt.month, dt.day)

def dbf2str(dbf_str):
    """
    Converts date from dbf-date to string (DD.MM.YYYY)

    Args:
        `dbf_str`:
            dbf-date (bytes in format YYYYMMDD)
    """
    if dbf_str is None or not dbf_str.isdigit() or len(dbf_str) != 8:
        result = None
    else:
        string_date = dbf_str.decode(SYSTEM_ENCODING)
        result = ".".join(
            reversed((string_date[:4], string_date[4:6], string_date[6:8]))
        )
    return result

def str2dbf(dt_str):
    """
    Converts from string to dbf-date (string in format YYYYMMDD)

    Args:
        `dt_str`:
            string in format DD.MM.YYYY
    """
    if not isinstance(dt_str, str):
        raise TypeError("Espects string or unicode instead of %s" % type(dt_str))
    str_l = len(dt_str)
    if str_l != 10:
        raise ValueError(
            "Datestring must be 10 symbols (DD.MM.YYYY) " "length instead of %d" % str_l
        )
    d, m, y = dt_str.split(".")
    dbf_string = "".join((y, m, d))
    return dbf_string.encode(SYSTEM_ENCODING)

# References:
# [dbfspec]: http://www.clicketyclick.dk/databases/xbase/format/index.html


import os
import pypdfium2 as pdfium

files = []
for file in os.listdir('.'):
    #split the file name and get the extension
    extension = os.path.splitext(file)[1]
    if extension == '.pdf':
        files.append(os.path.abspath(file))

    # Load a pdf document
for filepath in files:
    filename = os.path.basename(filepath)
    filename = os.path.splitext(filename)[0]
    pdf = pdfium.PdfDocument(filepath)
    page_indices = [i for i in range(len(pdf))]
    renderer = pdf.render_to(pdfium.BitmapConv.pil_image, page_indices = page_indices)
    if not os.path.exists(filename):
        #make a directory to store the images with the same name as the pdf file
        os.mkdir(filename)
        #check if the current directory is the temporary directory
    if os.getcwd() != filename:
        #change directory to the newly created directory
        os.chdir(filename)

    for image, index in zip(renderer, page_indices):
        #save the image
        image.save("output_%02d.jpg" % index)

    #change directory to the parent directory
    os.chdir('..')


import math
import time
from collections import OrderedDict

import numpy as np
import torch
import torch.nn as nn
import torch.nn.functional as F

import utils
from agent.sarsa import Sarsa

class GeneratorB(nn.Module):
    def __init__(self, tau_dim, skill_dim, hidden_dim):
        super().__init__()
        self.skill_pred_net = nn.Sequential(nn.Linear(tau_dim, hidden_dim),
                                            nn.ReLU(),
                                            nn.Linear(hidden_dim, hidden_dim),
                                            nn.ReLU(),
                                            nn.Linear(hidden_dim, skill_dim))

        self.apply(utils.weight_init)

    def forward(self, tau):
        skill_pred = self.skill_pred_net(tau)
        return skill_pred

class Discriminator(nn.Module):
    def __init__(self, tau_dim, feature_dim, hidden_dim):
        super().__init__()
        # def SimClR :
        self.embed = nn.Sequential(nn.Linear(tau_dim, hidden_dim),
                                   nn.ReLU(),
                                   nn.Linear(hidden_dim, hidden_dim),
                                   nn.ReLU(),
                                   nn.Linear(hidden_dim, feature_dim))

        self.project_head = nn.Sequential(nn.Linear(feature_dim, hidden_dim),
                                          nn.ReLU(),
                                          nn.Linear(hidden_dim, feature_dim))
        self.apply(utils.weight_init)

    def forward(self, tau):
        features = self.embed(tau)
        features = self.project_head(features)
        return features

class OURCAgent(Sarsa):
    def __init__(self, contrastive_scale,
                 update_encoder, contrastive_update_rate,skill_dim, temperature, update_skill_every_step,
                 **kwargs):
        self.skill_dim = skill_dim
        kwargs["meta_dim"] = self.skill_dim
        super().__init__(**kwargs)
        self.contrastive_scale = contrastive_scale
        self.update_encoder = update_encoder
        self.batch_size = kwargs['batch_size']
        self.contrastive_update_rate = contrastive_update_rate
        self.temperature = temperature

        self.update_skill_every_step = update_skill_every_step
        self.tau_len = update_skill_every_step

        # create ourc
        self.gb = GeneratorB(self.obs_dim - self.skill_dim, self.skill_dim,
                             kwargs['hidden_dim']).to(kwargs['device'])

        self.discriminator = Discriminator(self.obs_dim - self.skill_dim,
                                           self.skill_dim,
                                           kwargs['hidden_dim']).to(kwargs['device'])

        # loss criterion
        self.gb_criterion = nn.CrossEntropyLoss()
        self.discriminator_criterion = nn.CrossEntropyLoss()

        # optimizers
        self.gb_opt = torch.optim.Adam(self.gb.parameters(), lr=self.lr)
        self.dis_opt = torch.optim.Adam(self.discriminator.parameters(), lr=self.lr)

        self.gb.train()
        self.discriminator.train()

        self.skill_ptr = 0

    def init_meta(self):
        skill = np.zeros(self.skill_dim, dtype=np.float32)
        skill[self.skill_ptr] = 1
        meta = OrderedDict()
        meta['skill'] = skill
        return meta

    def update_meta(self, meta, global_step, time_step, finetune=False):
        if global_step % self.update_skill_every_step == 0:
            self.skill_ptr = (self.skill_ptr + 1) % self.skill_dim
            return self.init_meta()
        return meta

    def update_gb(self, skill, gb_batch, step):
        metrics = dict()
        labels = torch.argmax(skill, dim=1)
        loss, df_accuracy = self.compute_gb_loss(gb_batch, labels)

        self.gb_opt.zero_grad()
        loss.backward()
        self.gb_opt.step()

        return metrics

    def update_contrastive(self, taus, skills):
        metrics = dict()
        features = self.discriminator(taus)
        loss = self.compute_info_nce_loss(features, skills)
        loss = torch.mean(loss)
        metrics['loss'] = loss.item()
        self.dis_opt.zero_grad()
        loss.backward()
        self.dis_opt.step()

        return metrics

    def compute_intr_reward(self, skills, tau_batch, metrics):

        # compute q(z | tau) reward
        d_pred = self.gb(tau_batch)
        d_pred_log_softmax = F.log_softmax(d_pred, dim=1)
        _, pred_z = torch.max(d_pred_log_softmax, dim=1, keepdim=True)
        gb_reward = d_pred_log_softmax[torch.arange(d_pred.shape[0]), torch.argmax(skills, dim=1)] - math.log(
            1 / self.skill_dim)
        gb_reward = gb_reward.reshape(-1, 1)

        # compute contrastive reward
        features = self.discriminator(tau_batch)

        # maximize softmax item
        contrastive_reward = torch.exp(-self.compute_info_nce_loss(features, skills))
        intri_reward = gb_reward + contrastive_reward * self.contrastive_scale
        return intri_reward

    def compute_info_nce_loss(self, features, skills):

        size = features.shape[0] // self.skill_dim

        labels = torch.argmax(skills, dim=1)
        labels = (labels.unsqueeze(0) == labels.unsqueeze(1)).long()
        labels = labels.to(self.device)

        features = F.normalize(features, dim=1)

        similarity_matrix = torch.matmul(features, features.T)

        # discard the main diagonal from both: labels and similarities matrix
        mask = torch.eye(labels.shape[0], dtype=torch.bool).to(self.device)
        labels = labels[~mask].view(labels.shape[0], -1)
        similarity_matrix = similarity_matrix[~mask].view(similarity_matrix.shape[0], -1)
        similarity_matrix = torch.exp(similarity_matrix / self.temperature)

        # don't limit update for all negative
        pick_one_positive_sample_idx = torch.argmax(labels, dim=-1, keepdim=True)
        pick_one_positive_sample_idx = torch.zeros_like(labels).scatter_(-1, pick_one_positive_sample_idx, 1)
        neg = (~labels.bool()).long()
        # select one and combine multiple positives
        positives = torch.sum(similarity_matrix * pick_one_positive_sample_idx, dim=-1, keepdim=True)
        negatives = torch.sum(similarity_matrix * neg, dim=-1, keepdim=True)

        loss = -torch.log(positives / negatives)

        return loss

    def compute_gb_loss(self, taus, skill):
        """
        DF Loss
        """
        d_pred = self.gb(taus)
        d_pred_log_softmax = F.log_softmax(d_pred, dim=1)
        _, pred_z = torch.max(d_pred_log_softmax, dim=1, keepdim=True)
        d_loss = self.gb_criterion(d_pred, skill)
        df_accuracy = torch.sum(
            torch.eq(skill,
                     pred_z.reshape(1,
                                    list(
                                        pred_z.size())[0])[0])).float() / list(
            pred_z.size())[0]
        return d_loss, df_accuracy

    def update(self, buffer, step):
        metrics = dict()
        start = time.time()
        if step % self.update_every_steps != 0:
            return metrics

        batch = buffer.sample_batch(1024)
        obs, next_obs, action, rew, done, skill, next_skill = utils.to_torch(batch.values(), self.device)
        metrics.update(self.update_contrastive(next_obs, skill))
        for i in range(self.contrastive_update_rate - 1):
            batch = buffer.sample_batch(1024)
            obs, next_obs, action, rew, done, skill, next_skill = utils.to_torch(batch.values(), self.device)
            metrics.update(self.update_contrastive(next_obs, skill))

        # update q(z | tau)
        # bucket count for less time spending
        metrics.update(self.update_gb(skill, next_obs, step))

        # compute intrinsic reward
        with torch.no_grad():
            intr_reward = self.compute_intr_reward(skill, next_obs, metrics)
        metrics["reward"] = intr_reward.mean().item()
        if not self.update_encoder:
            obs = obs.detach()
            next_obs = next_obs.detach()

        action = action.cpu().numpy().astype('int').flatten()
        obs = obs.cpu().numpy().astype('int').flatten()
        next_obs = next_obs.cpu().numpy().astype('int').flatten()
        next_action = self.act(next_obs, skill).flatten()
        skill = torch.argmax(skill, dim=1).cpu().numpy()
        intr_reward = intr_reward.cpu().numpy().flatten()
        td_error = intr_reward + self.gamma * self.Q_table[next_obs, skill, next_action] - \
                   self.Q_table[obs, skill, action]
        self.Q_table[obs, skill, action] += self.alpha * td_error
        end = time.time()
        if step % 10000 == 0:
            print("on step : ",step, metrics, "update_time:", end-start)
        return metrics


from django import forms
from django.contrib.auth.forms import UserCreationForm
from django.contrib.auth.models import User
from .models import Thread, Post

class MySignUpForm(UserCreationForm):
    # imie_i_nazwisko = forms.CharField(max_length=100, required=False, help_text='Nie wymagane')
    first_name = forms.CharField(label='Imię', max_length=100, required=False, help_text='Nie wymagane')
    last_name = forms.CharField(label='Nazwisko', max_length=100, required=False, help_text='Nie wymagane')
    email = forms.EmailField(max_length=150)

    class Meta:
        model = User
        fields = ('username', 'first_name', 'last_name', 'email', 'password1', 'password2', )

    def __init__(self, *args, **kwargs):
        super(MySignUpForm, self).__init__(*args, **kwargs)

        self.fields['username'].help_text = 'Maksymalnie 150 znaków. Jedynie litery, cyfry i @/./+/-/_.'
        self.fields['password1'].help_text = 'Twoje hasło musi zawierać co najmniej 8 znaków. '

class ThreadForm(forms.ModelForm):

    class Meta:
        model = Thread
        fields = {'category_name', 'title', 'description'}

    def __init__(self, *args, **kwargs):
        super(ThreadForm, self).__init__(*args, **kwargs)
        self.fields['description'].widget.attrs['style'] = 'width:800px; height:150px'
        self.fields['title'].widget.attrs['style'] = 'width:600px; height:25px'
        self.fields['category_name'].widget.attrs['style'] = 'width:300px; height:25px'
        self.fields.keyOrder = ['title', 'category_name', 'description']
        self.fields['category_name'].label = 'Kategoria'
        self.fields['title'].label = 'Tytuł'
        self.fields['description'].label = 'Opis'

class PostForm(forms.ModelForm):

    class Meta:
        model = Post
        fields = {'text'}

    def __init__(self, *args, **kwargs):
        super(PostForm, self).__init__(*args, **kwargs)
        self.fields['text'].label = 'Komentarz'
        self.fields['text'].widget.attrs['cols'] = 100
        self.fields['text'].widget.attrs['rows'] = 15

class ProfileUpdateForm(forms.ModelForm):
    email = forms.EmailField(required=False, max_length=150)
    # imie_i_nazwisko = forms.CharField(required=False)
    first_name = forms.CharField(label='Imię', max_length=100, required=False, help_text='Nie wymagane')
    last_name = forms.CharField(label='Nazwisko', max_length=100, required=False, help_text='Nie wymagane')

    class Meta:
        model = User
        fields = ('email', 'first_name', 'last_name')


import copy
from abc import ABC, abstractmethod
from typing import Any, Callable, List, Optional, Set

import torch

from renate.models.layers import ContinualNorm
from renate.types import NestedTensors
from renate.utils.deepspeed import convert_to_tensor, recover_object_from_tensor

class RenateModule(torch.nn.Module, ABC):
    """A class for torch models with some additional functionality for continual learning.

    ``RenateModule`` derives from ``torch.nn.Module`` and provides some additional functionality
    relevant to continual learning. In particular, this concerns saving and reloading the model
    when model hyperparameters (which might affect the architecture) change during hyperparameter
    optimization. There is also functionality to retrieve internal-layer representations for use
    in replay-based CL methods.

    When implementing a subclass of ``RenateModule``, make sure to call the base class' constructor
    and provide your model's constructor arguments. Besides that, you can define a
    ``RenateModule`` just like ``torch.nn.Module``.

    Example::

        class MyMNISTMLP(RenateModule):

        def __init__(self, num_hidden: int):
            super().__init__(
                constructor_arguments={"num_hidden": num_hidden}
                loss_fn=torch.nn.CrossEntropyLoss()
            )
            self._fc1 = torch.nn.Linear(28*28, num_hidden)
            self._fc2 = torch.nn.Linear(num_hidden, 10)

        def forward(self, x: torch.Tensor) -> torch.Tensor:
            x = self._fc1(x)
            x = torch.nn.functional.relu(x)
            return self._fc2(x)

    The state of a ``RenateModule`` can be retrieved via the ``RenateModule.state_dict()`` method,
    just as in ``torch.nn.Module``. When reloading a ``RenateModule`` from a stored state dict, use
    ``RenateModule.from_state_dict``. It wil automatically recover the hyperparameters and
    reinstantiate your model accordingly.

    Note: Some methods of ``RenateModule`` accept an optional ``task_id`` argument. This is in
    anticipation of future methods for continual learning scenarios where task identifiers are
    provided. It is currently not used.

    Args:
        constructor_arguments: Arguments needed to instantiate the model.
    """

    def __init__(self, constructor_arguments: dict):
        super(RenateModule, self).__init__()
        self._constructor_arguments = copy.deepcopy(constructor_arguments)
        self._tasks_params_ids: Set[str] = set()
        self._intermediate_representation_cache: List[torch.Tensor] = []
        self._hooks: List[Callable] = []

    @classmethod
    def from_state_dict(cls, state_dict):
        """Load the model from a state dict.

        Args:
            state_dict: The state dict of the model. This method works under the assumption that
                this has been created by `RenateModule.state_dict()`.
        """
        extra_state = recover_object_from_tensor(state_dict["_extra_state"])
        constructor_arguments = extra_state["constructor_arguments"]
        model = cls(**constructor_arguments)
        for task in extra_state["tasks_params_ids"]:
            model.add_task_params(task)
        # TODO: See https://github.com/awslabs/Renate/issues/236.
        # There are changes to the `class_means` or `componenets` of a model
        # that are not loaded, and should probably not be stored.
        model.load_state_dict(state_dict, strict=False)
        return model

    def get_extra_state(self, encode: bool = True) -> Any:
        """Get the constructor_arguments, and task ids necessary to reconstruct the model."""
        extra_state = {
            "constructor_arguments": self._constructor_arguments,
            "tasks_params_ids": self._tasks_params_ids,
        }
        return convert_to_tensor(extra_state) if encode else extra_state

    def set_extra_state(self, state: Any, decode: bool = True):
        """Extract the content of the ``_extra_state`` and set the related values in the module."""
        extra_state = recover_object_from_tensor(state) if decode else state
        self._constructor_arguments = extra_state["constructor_arguments"]
        self._tasks_params_ids = extra_state["tasks_params_ids"]

    @abstractmethod
    def forward(self, x: NestedTensors, task_id: Optional[str] = None) -> torch.Tensor:
        """Performs a forward pass on the inputs and returns the predictions.

        This method accepts a task ID, which may be provided by some continual learning scenarios.
        As an example, the task id may be used to switch between multiple output heads.

        Args:
            x: Input(s) to the model. Can be a single tensor, a tuple of tensor, or a dictionary
                mapping strings to tensors.
            task_id: The identifier of the task for which predictions are made.
        Returns:
            The model's predictions.
        """
        pass

    def get_params(self, task_id: Optional[str] = None) -> List[torch.nn.Parameter]:
        """User-facing function which returns the list of parameters.

        If a ``task_id`` is given, this should return only parameters used for the specific task.

        Args:
            task_id: The task id for which we want to retrieve parameters.
        """
        return list(self.parameters())

    def _add_task_params(self, task_id: str) -> None:
        """Adds new parameters, associated to a specific task, to the model.

        The method should not modify modules created in previous calls, beyond the ones defined
        in ``self._add_task_params()``. The order of the calls is not guaranteed when the model
        is loaded after being saved.

        Args:
            task_id: The task id for which the new parameters are added.
        """
        pass

    def add_task_params(self, task_id: Optional[str] = None) -> None:
        """Adds new parameters, associated to a specific task, to the model.

        This function should not be overwritten; use ``_add_task_params`` instead.

        Args:
            task_id: The task id for which the new parameters are added.
        """
        if task_id in self._tasks_params_ids:
            return
        self._add_task_params(task_id)
        self._tasks_params_ids.add(task_id)

    def get_logits(self, x: NestedTensors, task_id: Optional[str] = None) -> torch.Tensor:
        """Returns the logits for a given pair of input and task id.

        By default, this method returns the output of the forward pass. This may be overwritten
        with custom behavior, if necessary.

        Args:
            x: Input(s) to the model. Can be a single tensor, a tuple of tensor, or a dictionary
                mapping strings to tensors.
            task_id: The task id.
        """
        return self.forward(x, task_id)

    def get_intermediate_representation(self) -> List[torch.Tensor]:
        """Returns the cached intermediate representation."""
        return self._intermediate_representation_cache

    def replace_batch_norm_with_continual_norm(self, num_groups: int = 32) -> None:
        """Replaces every occurence of batch normalization with continual normalization.

        Pham, Q., Liu, C., & Hoi, S. (2022). Continual normalization: Rethinking batch
        normalization for online continual learning. arXiv preprint arXiv:2203.16102.

        Args:
            num_groups: Number of groups when considering the group normalization in continual
                normalization.
        """

        def _replace(module):
            for name, child in module.named_children():
                if not list(module.children()):
                    _replace(child)
                if isinstance(
                    child, (torch.nn.BatchNorm1d, torch.nn.BatchNorm2d, torch.nn.BatchNorm3d)
                ):
                    setattr(
                        module,
                        name,
                        ContinualNorm(
                            num_features=child.num_features,
                            eps=child.eps,
                            momentum=child.momentum,
                            affine=child.affine,
                            track_running_stats=child.track_running_stats,
                            num_groups=num_groups,
                        ),
                    )

        _replace(self)

    def _intermediate_representation_caching_hook(self) -> Callable:
        """Hook to cache intermediate representations during training."""

        def hook(m: torch.nn.Module, _, output: torch.Tensor) -> None:
            if m.training:
                self._intermediate_representation_cache.append(output)

        return hook

    def register_intermediate_representation_caching_hook(self, module: torch.nn.Module) -> None:
        """Add a hook to cache intermediate representations during training.

        Store the reference to the hook to enable its removal.

        Args:
            module: The module to be hooked.
        """
        hook = module.register_forward_hook(self._intermediate_representation_caching_hook())
        self._hooks.append(hook)

    def deregister_hooks(self) -> None:
        """Remove all the hooks that were registered."""
        for hook in self._hooks:
            hook.remove()
        self._hooks = []
        self.reset_intermediate_representation_cache()

    def reset_intermediate_representation_cache(self) -> None:
        """Resets the intermediate representation cache."""
        self._intermediate_representation_cache = []

class RenateWrapper(RenateModule):
    """A simple wrapper around a torch model.

    If you are using a torch model with fixed hyperparameters, you can use this wrapper to expose
    it as a ``RenateModule``. In this case, do _not_ use the ``from_state_dict`` method but
    reinstantiate the model, wrap it, and call ``load_state_dict``. If a tuple or a dictionary of
    tensors is passed to the `RenateWrapper`'s forward function, it is unpacked before passing it
    to the torch model's forward function.

    Example::

        my_torch_model = torch.nn.Linear(28*28, 10)  # Instantiate your torch model.
        model = RenateWrapper(my_torch_model)
        state_dict = torch.load("my_state_dict.pt")
        model.load_state_dict(state_dict)

    Args:
        model: The torch model to be wrapped.
    """

    def __init__(self, model: torch.nn.Module) -> None:
        super().__init__(constructor_arguments={})
        self._model = model

    def forward(self, x: NestedTensors, task_id: Optional[str] = None) -> torch.Tensor:
        if isinstance(x, torch.Tensor):
            outputs = self._model(x)
        elif isinstance(x, tuple):
            outputs = self._model(*x)
        elif isinstance(x, dict):
            outputs = self._model(**x)
        else:
            raise TypeError(f"Expected tensor or tuple/dict of tensors; found {type(x)}.")

        if isinstance(outputs, tuple) and len(outputs) == 1:
            return outputs[0]
        else:
            return outputs

    @classmethod
    def from_state_dict(cls, state_dict):
        raise NotImplementedError(
            "RenateWrapper does not support `from_state_dict`. Instantiate the object using the "
            "standard constructor, then call `load_state_dict`."
        )


"""
Created on October 19, 2018

@author: mae-ma
@attention: evaluation of the architectures
@contact: albus.marcel@gmail.com (Marcel Albus)
@version: 1.3.1

#############################################################################################

History:
- v1.3.1: cleanup
- v1.3.0: plot for q-vals
- v1.2.1: change filenames
- v1.2.0: use smoothed score output for better visualization
- v1.1.1: use relative paths
- v1.1.0: add click commands
- v1.0.0: first init
"""

import numpy as np
from matplotlib import pyplot as plt
import pickle
import yaml
import shutil
import os
import click
import datetime

def smooth(x, window_len=11, window='hanning'):
    """smooth the data using a window with requested size.

    This method is based on the convolution of a scaled window with the signal.
    The signal is prepared by introducing reflected copies of the signal
    (with the window size) in both ends so that transient parts are minimized
    in the begining and end part of the output signal.

    input:
        x: the input signal
        window_len: the dimension of the smoothing window; should be an odd integer
        window: the type of window from 'flat', 'hanning', 'hamming', 'bartlett', 'blackman'
            flat window will produce a moving average smoothing.

    output:
        the smoothed signal

    example:

    t=linspace(-2,2,0.1)
    x=sin(t)+randn(len(t))*0.1
    y=smooth(x)

    see also:

    numpy.hanning, numpy.hamming, numpy.bartlett, numpy.blackman, numpy.convolve
    scipy.signal.lfilter

    TODO: the window parameter could be the window itself if an array instead of a string
    NOTE: length(output) != length(input), to correct this: return y[(window_len/2-1):-(window_len/2)] instead of just y.
    """

    if x.ndim != 1:
        raise ValueError("smooth only accepts 1 dimension arrays.")

    if x.size < window_len:
        raise ValueError("Input vector needs to be bigger than window size.")

    if window_len < 3:
        return x

    if not window in ['flat', 'hanning', 'hamming', 'bartlett', 'blackman']:
        raise ValueError(
            "Window is on of 'flat', 'hanning', 'hamming', 'bartlett', 'blackman'")

    s = np.r_[x[window_len-1:0:-1], x, x[-1:-window_len:-1]]
    # print(len(s))
    if window == 'flat':  # moving average
        w = np.ones(window_len, 'd')
    else:
        w = eval('np.'+window+'(window_len)')

    y = np.convolve(w/w.sum(), s, mode='valid')
    return y

class Evaluation:
    def __init__(self):
        # src_filepath = home/mae-ma/git/safety
        # self.src_filepath = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
        self.src_filepath = os.getcwd()
        self.plot_filename = None

        with open(os.path.join(self.src_filepath, 'reward.yml'), 'r') as file:
            self.reward = yaml.load(file)
        with open(os.path.join(os.path.join(self.src_filepath, 'architectures'), 'config_dqn.yml'), 'r') as file:
            self.dqn_config = yaml.load(file)
        with open(os.path.join(self.src_filepath, 'model.yml'), 'r') as file:
            self.model = yaml.load(file)

        self.tgt_filepath = os.path.join(os.path.dirname(
            os.path.dirname(__file__)), 'results')
        if not os.path.exists(self.tgt_filepath):
            os.makedirs(self.tgt_filepath)

    def plot(self):
        print('–'*50)
        print('Plot "reward.yml"')
        csv_path = os.path.join(os.getcwd(), 'training_log_DQN.csv')
        self.csv = np.genfromtxt(csv_path, delimiter=',')
        smoothed = smooth(self.csv[:, 2], 31)

        fig, (ax1, ax2) = plt.subplots(2, 1, figsize=(12, 12))
        ax1.plot([_ for _ in range(len(smoothed))],
                 smoothed, 'b', label='loss')
        ax1.set_ylabel('Loss')

        score_value = [x[0] for x in self.reward]
        score_time = [x[1] for x in self.reward]
        ax2.plot(score_time, smooth(np.array(score_value), 11)[:-10], 'r', label='scores')
        ax2.set_xlabel('Steps')
        ax2.set_ylabel('Scores')
        ax1.set_xlim([- len(smoothed)*.05, len(smoothed)*1.05])
        ax2.set_xlim([- len(smoothed)*.05, len(smoothed)*1.05])
        ax2.legend()
        ax1.legend()
        ax2.grid()
        ax1.grid()
        fig.tight_layout()
        if self.model['config'][0]['class_name'] == 'Conv2D':
            model = 'Conv2D'
        else:
            model = '-u' + str(self.model['config'][0]['config']['units'])
        filename = 'lr' + \
            str(self.dqn_config['learning_rate']).replace('.', '_') + \
            '-g' + str(self.dqn_config['gamma']).replace('.', '_') + \
            model + '.pdf'
        self.plot_filename = filename
        plt.savefig(filename)
        plt.show()
        print('–'*50)

    def plot_q_vals(self):
        csv_path = os.path.join(os.getcwd(), 'q_val_DQN.csv')
        self.csv = np.genfromtxt(csv_path, delimiter=',')
        smoothed = smooth(self.csv[:], 51)
        fig, ax1 = plt.subplots(1, 1, figsize=(12, 12))
        ax1.plot([_ for _ in range(len(smoothed))],
                 smoothed, 'b', label='Q values')
        ax1.set_ylabel('Q values')
        ax1.set_xlabel('Steps')
        plt.legend()
        plt.grid()
        # plt.show()

    def save_all(self):
        print('–'*50)
        filelist = ['weights.h5', 'target_weights.h5',
                    'reward.yml', 'replay_buffer.pkl', 'training_log_DQN.csv',
                    self.plot_filename, 'architectures/config_dqn.yml', 'model.yml']
        folder = datetime.datetime.today().strftime('%Y_%m_%d-%H_%M') + '___' + self.plot_filename
        folderpath = os.path.join(self.tgt_filepath, folder)
        print('Save all files to: ' + folderpath)
        if not os.path.exists(folderpath):
            os.makedirs(folderpath)
        for file in filelist:
            shutil.copy2(os.path.join(self.src_filepath, file), folderpath)
        print('–'*50)

@click.command()
@click.option('--plot/-no-plot', '-p/-np', default=True, help='plot the results from the "results.yaml" file')
@click.option('--save/--no-save', '-s/-ns', default=False, help='backups the files')
def main(plot, save):
    ev = Evaluation()
    print('src: ', ev.src_filepath)
    if plot:
        ev.plot()
        # ev.plot_q_vals()
    if save:
        ev.save_all()

if __name__ == '__main__':
    main()


from flask import Flask, request, jsonify
from flask_cors import CORS
from pymongo import MongoClient
import subprocess
import json
import os

app = Flask(__name__)
CORS(app)

# Load the config from config.json
with open('config/config.json') as config_file:
    config_data = json.load(config_file)

# Connect to the MongoDB database
client = MongoClient(config_data['database'])
db = client['spotify']
collection = db['search_input']

@app.route('/search-artist', methods=['POST'])
def search_artist():
    artist_name = request.json['artistName']
    print(artist_name)

    # Update the search_input document with the artist name
    collection.update_one({}, {"$set": {"searchInput": artist_name}}, upsert=True)

    # Execute the data.js script
    subprocess.run(['node', 'data.js'])

    response = {
        'artistName': artist_name
    }
    return jsonify(response)

if __name__ == '__main__':
    # Use the PORT environment variable provided by Heroku
    port = int(os.environ.get(5000))
    app.run(host='0.0.0.0', port=port)


import numpy as np
import matplotlib.pyplot as plt

def main():

    N_bandits = 10
    bandits_probs = abs(np.random.normal(size=N_bandits))
    N_experiments = 100
    N_episodes = 100

    class Bandit:
        def __init__(self,bandits_probs):
            self.N = len(bandits_probs)
            self.probs = bandits_probs

        def get_reward(self,action):
            rand = abs(np.random.normal())
            reward = 1 if (rand > self.probs[action]) else 0
            return reward

    class Agent:
        def __init__(self,bandit):
            self.k = np.zeros(bandit.N,dtype=np.int)
            self.Q = np.zeros(bandit.N,dtype=np.float)

        def update_Q(self,action,reward):
            self.k[action] +=1
            self.Q[action] += (reward - self.Q[action])/self.k[action]

    def experiment(agent,bandit,N_episodes):

        action_history = []
        reward_history = np.zeros(bandit.N)
        for i in range(N_episodes):
            for j in range(N_bandits):
                reward = bandit.get_reward(j)

                agent.update_Q(j,reward)

                #action_history.append(action)
                reward_history[j] = reward

        print(reward_history.shape)
        return np.array(reward_history)

    reward_history_avg = np.zeros(N_bandits)

    for i in range(N_experiments):
        bandit = Bandit(bandits_probs)
        agent = Agent(bandit)

        reward_history = experiment(agent,bandit,N_episodes)

        reward_history_avg += reward_history

    reward_history_avg /=np.float(N_experiments)
    best_arm = np.argmax(reward_history_avg)
    print("Reward history avg ={}".format(reward_history_avg))
    print("The Best arm to choose after {} experiments is {} with a reward of {}".format(N_experiments,best_arm,reward_history_avg.max()))

    plt.plot(reward_history_avg)
    plt.show()

main()


# PARKING MANAGEMENT #
#
import datetime
#
# For insurance purposes, the management of an office building is required to
# maintain, at all time, an accurate list of all the vehicles in the dedicated
# parking. In addition, for billing the different companies, the office
# building management wants to record occupation of the parking at different
# times and automatically emit bills to each specific companies.
#
# You are tasked with completing the series of functions below that fill the
# need of the office building parking management. You are allowed (and
# encouraged) to create additional, intermediate functions.
#
# The main data structure that your suite of function handles is a record of
# entrances and exits. A sample is given below. It consist of a pair of lists
# of tuples. The first list gives the timestamps and license plate number of
# vehicles entering the parking, the second exiting.
#
# DO NOT MODIFY CONSTANTS
PARKING_DATA_SAMPLE = ([
    (datetime.datetime(2017, 12, 12, 7, 13, 44, 0), 'LR10GHT'),
    (datetime.datetime(2017, 12, 12, 7, 13, 48, 0), 'LC11FBF'),
    (datetime.datetime(2017, 12, 12, 7, 13, 59, 0), 'LR10ZPP'),
    (datetime.datetime(2017, 12, 12, 7, 15, 2, 0), 'LJ65OSN'),
    (datetime.datetime(2017, 12, 12, 7, 15, 22, 0), 'LA63EWH'),
    (datetime.datetime(2017, 12, 12, 13, 1, 42, 0), 'LC11FBF')
], [(datetime.datetime(2017, 12, 12, 12, 13, 1, 0), 'LC11FBF'),
    (datetime.datetime(2017, 12, 12, 16, 42, 10, 0), 'LR10ZPP'),
    (datetime.datetime(2017, 12, 12, 17, 2, 41, 0), 'LR10GHT'),
    (datetime.datetime(2017, 12, 12, 17, 2, 58, 0), 'LA63EWH'),
    (datetime.datetime(2017, 12, 12, 17, 4, 3, 0), 'LJ65OSN'),
    (datetime.datetime(2017, 12, 12, 17, 10, 21, 0), 'LC11FBF')])
#
# A secondary data structure includes billing information. It is a dictionary
# that maps company names to a list of registered license plates.
#
# DO NOT MODIFY CONSTANTS
COMPANY_REGISTRATIONS_SAMPLE = {
    'Shire Tobacco Inc.': ['LR10GHT', 'LA63EWH'],
    'Rohan Equestrian Equipments': [],
    'Moria Construction Hardware': ['LC11FBF', 'LS66XKE', 'LR10ZPP', 'LJ65OSN']
}

def register_car(registration, company, plate):
    """
    Registers a new car.

    NOTE: this function should not modify the registration dictionary that is
    given, instead it should create a new dictionary.
    NOTE: this function should not introduce duplicates in the registration
    system. Specifically, if a car is already registered with the given
    company it should return an identical registration information. If the car
    is registered with a different company it should remove the first
    registration.
    NOTE: if the company is not listed in the dictionary, it should not
    introduce it. Instead it should just return an identical registration.

    E.g., register_car({'Stark Industries': ['IRNMN']}, 'Stark Industries',
                       'JARVIS')
    is {'Stark Industries': ['IRNMN', 'JARVIS']}
    E.g., register_car({'Stark Industries': ['IRNMN']}, 'Wayne Enterprises',
                       'IMBTMN')
    is {'Stark Industries': ['IRNMN']}

    :param registration: preexisting registration information
    :param company: company to register the car for
    :param plate: license plate of the car to register
    :return: new registration information dictionary with added registration
    :rtype: dict
    """
    l_registration = registration
    for key, value in l_registration.items():
        if company != key:
            if plate in [x for v in l_registration.values() for x in v]:
                try:
                    l_registration[key].remove(plate)
                except ValueError:
                    a = 0

    for key, value in l_registration.items():
        if company == key:
            if plate not in [x for v in l_registration.values() for x in v]:
                l_registration[key].append(plate)

    return l_registration

register_car({'Stark Industries': ['IRNMN']}, 'Stark Industries', 'JARVIS')
register_car({'Stark Industries': ['IRNMN']}, 'Wayne Enterprises', 'IMBTMN')
register_car({'X': []}, 'X', 'A')
register_car({'X': [], 'Y': ['A']}, 'X', 'A')
register_car({'Stark Industries': ['IRNMN']}, 'Stark Industries', 'JARVIS')

def occupancy(parking_data, cutoff_time=None):
    """
    Computes the occupancy of the parking at a given time. If no time is
    provided, check the current occupancy.

    E.g.,
    data = ([(datetime.datetime(2017, 12, 12,  7, 13, 44, 0), 'LR10GHT')], [])
    occupancy(data, time=datetime.datetime(2017, 12, 12,  7, 13, 45, 0))
    is ['LR10GHT']
    E.g.,
    data = ([(datetime.datetime(2017, 12, 12,  7, 13, 44, 0), 'LR10GHT')], [])
    occupancy(data, time=datetime.datetime(2017, 12, 12,  7, 13, 43, 0))
    is []

    :param parking_data: tuple of list of timestamped arrival and departure
    information including license plate. See sample above.
    :param time: time (as a datetime.datetime object) at which to check for
    occupancy. If no time is provided, use now.
    :return: list of cars present in the parking at the given time.
    :rtype: list
    """

    l_entry = parking_data[0]
    l_exit = parking_data[1]
    l_time = cutoff_time
    l_entered = {}
    l_plates = list([])

    if l_time is None:
        l_time = datetime.datetime.now()
    for entry in l_entry:
        print(entry)
        for i in entry:
            print(i)
            l_entrytime = entry[0]
            l_plate = entry[1]
            if l_plate not in l_entered:
                l_entered[l_plate] = [l_entrytime]
                print(l_entered)

    for exits in l_exit:
        print(exit)
        for i in exits:
            print("exit {}".format(i))
            l_exittime = exits[0]
            l_plate = exits[1]
        if l_plate not in l_entered:
            l_entered[l_plate] = [l_exittime]
        else:
            l_entered[l_plate].append(l_exittime)

    for key, value in l_entered.items():
        print(key)
        for index,val in enumerate(value):
            if index % 2 == 0:
                l_entry = value[index]
                try:
                    l_exit = value[index+1]
                except IndexError:
                    l_exit = datetime.datetime.now()
                print(l_entry, l_exit)
    if l_exit != [] and l_exit >= l_time:
        l_plates.append(key)

    return l_plates

data = ([(datetime.datetime(2017, 12, 12,  7, 13, 44, 0), 'LR10GHT')], [])
cutoff_time = datetime.datetime(2000, 1, 3, 12, 0)
occupancy(data, cutoff_time)
data = ([(datetime.datetime(2017, 12, 12,  7, 13, 44, 0), 'LR10GHT')], [])
cutoff_time = datetime.datetime(2000, 1, 3, 12, 0)
occupancy(data, cutoff_time)

def company_bill(parking_data, company_registration, company, t_start, t_end):
    """
    Computes the total, cumulated time in seconds, ignoring milliseconds, that
    cars registred with a given company stayed in the parking during the
    interval between t_start and t_end.

    E.g.,
    parking_data = (
        [(datetime.datetime(2017, 12, 12,  7, 13, 44, 0), 'LR10GHT')],
        [(datetime.datetime(2017, 12, 12,  7, 13, 45, 0), 'LR10GHT')]
    )
    company_registration = {'Shire Tobacco Inc.': ['LR10GHT']}
    company_bill(parking_data, company_registration,
                 'Shire Tobacco Inc.', …, …)
    is 1

    :param parking_data: see sample above
    :param company_registration: see sample above
    :param company: name of the company to compute billing for
    :param t_start: start of the billing interval
    :param t_end: end of the billing interval
    :return: cumulated number of seconds of car park occupancy
    :rtype: float | int
    """
    l_entry = parking_data[0]
    l_exit = parking_data[1]
    l_seconds = 0
    l_enttime = None
    l_exittime = None
    for l_company, l_plate in company_registration.items():
        if l_company != company:
            break  # do not continue
        l_plates = [x for x in l_plate]
        for plate in l_plates:
            for t in l_entry:
                l_ent = list(t)
                if plate in l_ent[1]:
                    l_enttime = l_ent[0]
                else:
                    l_enttime = None
            for e in l_exit:
                l_ex = list(e)
                if plate in l_ex:
                    l_exittime = l_ent[0]
                else:
                    l_exittime = None
        if (l_enttime is None or l_exittime is None):
            return l_seconds
        else:
            l_seconds = (l_exittime - l_enttime).seconds

    return l_seconds

parking_data = ([(datetime.datetime(2000, 1, 1, 12, 0), 'LR10GHT')],
[(datetime.datetime(2000, 1, 2, 12, 0), 'LR10GHT')])
start_time = datetime.datetime(2000, 1, 3, 12, 0)
end_time = datetime.datetime(2000, 1, 3, 12, 0)
company_registration = {'Shire Tobacco Inc.': ['LR10GHT']}
company_bill(parking_data, company_registration, 'Shire Tobacco Inc.', start_time, end_time)

parking_data = ([(datetime.datetime(2000, 1, 1, 12, 0), 'LR10GHT')],
    [(datetime.datetime(2000, 1, 2, 12, 0), 'LR10GHT')])
start_time = datetime.datetime(2000, 1, 3, 12, 0)
end_time = datetime.datetime(2000, 1, 3, 12, 0)
company_registration = {'Shire Tobacco Inc.': ['LR10GHT']}
company_bill(parking_data, company_registration, 'Shire Tobacco Inc.', start_time, end_time)

parking_data = ([(datetime.datetime(2000, 1, 1, 12, 0), 'LR10GHT')], [])
start_time = datetime.datetime(2000, 1, 3, 12, 0)
end_time = datetime.datetime(2000, 1, 3, 12, 0)
company_registration = {'Shire Tobacco Inc.': ['LR10GHT']}
company_bill(parking_data, company_registration, 'Shire Tobacco Inc.', start_time, end_time)

parking_data = ([(datetime.datetime(2000, 1, 1, 12, 0), 'LR10GHT'), (datetime.datetime(2000, 1, 2, 12, 0), 'LR10GHT'), (datetime.datetime(2000, 1, 3, 12, 0), 'LR10GHT')], [(datetime.datetime(2000, 1, 1, 12, 1), 'LR10GHT'), (datetime.datetime(2000, 1, 2, 12, 1), 'LR10GHT'), (datetime.datetime(2000, 1, 3, 12, 1), 'LR10GHT')])
start_time = datetime.datetime(2000, 1, 3, 12, 0)
end_time = datetime.datetime(2000, 1, 3, 12, 0)
company_registration = {'Shire Tobacco Inc.': ['LR10GHT']}
company_bill(parking_data, company_registration, 'Shire Tobacco Inc.', start_time, end_time)