import mimetypes
import os
import socket
import typing

SERVER_ROOT = "www"

HOST = "127.0.0.1"
PORT = 9000

FILE_RESPONSE_TEMPLATE = """\
HTTP/1.1 200 OK
Content-type: {content_type}
Content-length: {content_length}

"""

METHOD_NOT_ALLOWED_RESPONSE = b"""\
HTTP/1.1 405 Method Not Allowed
Content-type: text/plain
Content-length: 17

Method Not Allowed"""

NOT_FOUND_RESPONSE = b"""\
HTTP/1.1 404 Not Found
Content-type: text/plain
Content-length: 9

Not Found"""

BAD_REQUEST_RESPONSE = b"""\
HTTP/1.1 400 Bad Request
Content-type: text/plain
Content-length: 11

Bad Request"""

def serve_file(sock: socket.socket, path: str) -> None:
    """Given a socket and the relative path to a file (relative to
    SERVER_SOCK), send that file to the socket if it exists. If the
    file doesn't exist, send a "404 Not Found" response.
    """
    if path == "/":
        path = "/index.html"

    abspath = os.path.normpath(os.path.join(SERVER_ROOT, path.lstrip("/")))

    if not os.path.exists(abspath):
        sock.sendall(NOT_FOUND_RESPONSE)
        return

    with open(abspath, "rb") as f:
        stat = os.fstat(f.fileno())
        content_type, encoding = mimetypes.guess_type(abspath)
        if content_type is None:
            content_type = "application/octet-stream"

        if encoding is not None:
            content_type += f"; charset={encoding}"

        response_headers = FILE_RESPONSE_TEMPLATE.format(
            content_type=content_type, content_length=stat.st_size,
        ).encode("utf-8")

        sock.sendall(response_headers)
        sock.sendfile(f)

def iter_lines(
    sock: socket.socket, bufsize: int = 1024
) -> typing.Generator[bytes, None, bytes]:
    """Given a socket, read all the individual CRLF-separated lines
    and yield each one until an empty one is found. Returns the
    remainder after the empty line.
    """
    buff = b""
    while True:
        data = sock.recv(bufsize)
        if not data:
            return buff

        buff += data
        while True:
            try:
                i = buff.index(b"\r\n")
                line, buff = buff[:i], buff[i + 2 :]
                if not line:
                    return buff

                yield line
            except IndexError:
                break

class Request(typing.NamedTuple):
    method: str
    path: str
    headers: typing.Mapping[str, str]

    @classmethod
    def from_socket(cls, sock: socket.socket) -> "Request":
        """Read and parse the request from a socket object.

        Raises:
          ValueError: When the request cannot be parsed.
        """
        lines = iter_lines(sock)

        try:
            request_line = next(lines).decode("utf-8")
        except StopIteration:
            raise ValueError("Request line missing.")

        try:
            method, path, _ = request_line.split(" ")
        except ValueError:
            raise ValueError(f"Malformed request line {request_line!r}")

        headers = {}
        for line in lines:
            try:
                name, _, value = line.decode("utf-8").partition(":")
                headers[name.lower()] = value.lstrip()
            except ValueError:
                raise ValueError(f"Malformed header line {line!r}")

        return cls(method=method.upper(), path=path, headers=headers)

# By default, socket.socket creates TCP sockets
with socket.socket() as server_sock:
    server_sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
    server_sock.bind((HOST, PORT))
    server_sock.listen(0)
    print(f"Listening on {HOST}:{PORT}...")

    while True:
        client_sock, client_addr = server_sock.accept()
        print(f"New connection from {client_addr}")
        with client_sock:
            try:
                request = Request.from_socket(client_sock)
                if request.method != "GET":
                    client_sock.sendall(METHOD_NOT_ALLOWED_RESPONSE)
                    continue

                serve_file(client_sock, request.path)
            except Exception as e:
                print(f"Failed to parse request: {e}")
                client_sock.sendall(BAD_REQUEST_RESPONSE)


from django.shortcuts import redirect, render
def home(request):
    import xlrd
    workbook = xlrd.open_workbook("D:\\indhu college\\miniproject-2\\covid\\tamilnadu.xls")
    sheet = workbook.sheet_by_index(0)
    col=sheet.cell_value(0, 0)
    rows=sheet.nrows
    print(sheet.cell_value(22, 6))
    print(rows)
    cured=sheet.cell_value(rows-1,3)
    death=sheet.cell_value(rows-1,4)
    total_cases=sheet.cell_value(rows-1,5)
    cured=int(cured)
    death=int(death)
    total_cases=int(total_cases)
    print(cured)
    # Extracting number of columns
    print(sheet.ncols)
     #vaccine
    workbook1 = xlrd.open_workbook("D:\\indhu college\\miniproject-2\\covid\\vaccine.xls")
    sheet1=workbook1.sheet_by_index(0)
    n=sheet1.nrows
    vaccinated=sheet1.cell_value(n-1,20)
    vaccinated=int(vaccinated)
    return render(request,'home.html',{'cured':cured,'death':death,'total_cases':total_cases,'vaccinated':vaccinated})

def charts(request):
    import xlrd
    workbook = xlrd.open_workbook("D:\\indhu college\\miniproject-2\\covid\\tamilnadu.xls")
    workbook1 = xlrd.open_workbook("D:\\indhu college\\miniproject-2\\covid\\vaccine.xls")
    sheet = workbook.sheet_by_index(0)
    sheet1=workbook1.sheet_by_index(0)
    val=[]
    months=[]
    m=[]
    j=0
    #vaccine
    n=sheet1.nrows
    vaccinated=sheet1.cell_value(n-1,17)
    for i in range(0,sheet.nrows):
        if j!=0:
            val.append(int(sheet.cell_value(i,6)))
            xl_date=sheet.cell_value(i,0)
            datetime_date = xlrd.xldate_as_datetime(xl_date, 0)
            date_object = datetime_date.date()
            string_date = date_object.isoformat()
            #months.append(string_date[8:])
            m.append(string_date)
        j=j+1
    #print(months)
    val=list(map(str,val))
    val=' '.join(val)
    months=list(map(str,months))
    months=' '.join(months)
    return render(request,"charts.html",{'x':val,'y':m})

def covidend(request):
    import xlrd
    workbook1 = xlrd.open_workbook("D:\\indhu college\\miniproject-2\\covid\\vaccine.xls")
    sheet1=workbook1.sheet_by_index(0)
    n=sheet1.nrows
    noofpeopletoachieveheardimmunity=940000000*0.7
    vaccinated=sheet1.cell_value(n-1,19)
    totalvaccinated=int(vaccinated)
    vaccinatedpeople=int(totalvaccinated/2)
    noofpeoplevaccinatedinpercent=(vaccinatedpeople*100)/920000000
    balancepeople=(940000000*0.7)-vaccinatedpeople
    herdimmuntiyachievedsofar=(noofpeoplevaccinatedinpercent*90)/100

    return render(request,"covidend.html",{'vaccinatedpeople':vaccinatedpeople,'noofpeoplevaccinatedinpercent':noofpeoplevaccinatedinpercent,'balancepeople':balancepeople,'herdimmuntiyachievedsofar':herdimmuntiyachievedsofar,'herdimmunityneeded':63})


#! usr/bin/python3.5

import numpy as np
from numpy.linalg import inv
from scipy import linalg

def MyFirstAlgorithm(X,mode='cov',k=1, threshold=0.3):
	print('Now running conventional PCA through eigendecomposition of the covariance matrix...')
	if mode == 'cov':
		#Calculation of Covariance matrix
		C = np.cov(X)
	elif mode == 'noncov':
		#Calculation of the Scatter matrix
		meanM=[]
		for line in X:
			meanM.append([np.mean(line)])
		meanMatrix=np.array(meanM)
		C = np.array(sum([np.mat(x-meanMatrix).T.dot(np.mat(x-meanMatrix)) for x in X.T]))
	eigVals,eigVecs = linalg.eigh(C) #the eigenvalues are given in ascending order
	index=eigVals.argsort()[::-1]  #finding the indexes for descending order
	eigvecs=eigVecs[:k,index] #the first k vectors will be kept
	W=eigvecs[:]
	y=W.dot(X)
	return(y, eigVals)

def PCA(Data, k=1, F=True, threshold=0.02):
	print('Now running conventional PCA through singular value decomposition of the data...')
	D = Data.shape[0]
	N = Data.shape[1]
	meanM = []
	for line in Data:
		meanM.append([np.mean(line)])
	meanMatrix = np.array([meanM]*N).squeeze().T
	X = Data - meanMatrix #centered

	U,S,V = np.linalg.svd(X, full_matrices=F)
	eigvecs=U.T[:k] #the first k vectors will be kept
	W=eigvecs[:]
	y=W.dot(Data)
	return(y, S)

def PPCA(Data, M=1, variance=True):
	pie=np.pi
	D = Data.shape[0]
	N = Data.shape[1]
	meanM = []
	for line in Data:
		meanM.append([np.mean(line)])
	meanMatrix = np.array([meanM]*N).squeeze().T
	X = Data - meanMatrix #centered

	if variance==False: #Sigma is zero
		print('Now running PPCA (sigma zero) on the data...')
		#SET ARBITRARY W#
		mean = np.zeros(M)
		var = np.identity(M)
		W = np.random.multivariate_normal(mean,var,D) #DxM matrix
		while True:
			Mi = inv(W.T.dot(W))
			O = Mi.dot(W.T).dot(X) #O is the expectation E[x] where x are the observed values

			W_new = X.dot(O.T).dot(inv(O.dot(O.T)))

			if sum(sum(abs(W_new-W)**2)) < 0.000001:
				U,S,V = np.linalg.svd(W_new, full_matrices=False)
				Output = U.T.dot(Data)
				break
			else:
				W = W_new
		return (Output)

	if variance==True: #Sigma is not zero
		print('Now running PPCA (sigma not zero) on the data...')
		Runs=0
		Wfinal=np.zeros((D,M))
		def EM(X):
			#SET ARBITRARY W#
			W = np.random.rand(D,M)
			#SET ARBITRARY Sigma#
			sigma = np.random.random()
			metric = 0
			counter = 0
			while True:
				#In the following loops i will split the terms of W_new and sigma_new to separate variables to avoid a huge equation.
				counter+=1

				Mi = inv(W.T.dot(W) + sigma*np.identity(W.shape[1]))
				Ez = []
				Ezzt = []
				A = np.zeros((D,M)) #Starting with a zero matrix, to sum the first term of Wnew

				for x in X.T:
					x = np.mat(x)
					Ez_n = Mi.dot(W.T).dot(x.T)
					Ezzt_n = sigma*Mi + Ez_n.dot(Ez_n.T)
					Ezzt.append(Ezzt_n)
					A += x.T.dot(Ez_n.T)

				Ezzt = np.array(Ezzt) #Nx(MxM)
				B = inv(sum(Ezzt)) #second term of Wnew
				W_new = A.dot(B)

				values=[]
				for x in X.T:
					x = np.mat(x)
					Ez_n = Mi.dot(W.T).dot(x.T)
					a = x.dot(x.T)
					b = Ez_n.T.dot(W_new.T).dot(x.T)
					Ezzt_n = sigma*Mi + Ez_n.dot(Ez_n.T)
					c = np.trace(Ezzt_n.dot(W_new.T).dot(W_new))
					values.append(float(a - 2*b + c))

				sigma_new = (1/N*D)*sum(values)

				#We will use the joint p(x,z) expectation given the parameters as convergence metric
				#In each run we will compare with the previous
				#In the first run we have defined the old metric as 0 to skip right ahead.
				a = (D/2)*np.log(2*pie*sigma_new)
				b = []
				c = []
				d = []
				e = []
				for x in X.T:
					x = np.mat(x)
					Ez_n = Mi.dot(W.T).dot(x.T)
					Ezzt_n = sigma*Mi + Ez_n.dot(Ez_n.T)
					b.append(float(1/2)*np.trace(Ezzt_n))
					c.append(float((1/2*sigma_new)*(x.dot(x.T))))
					d.append(float((1/sigma_new)*Ez_n.T.dot(W_new.T).dot(x.T)))
					e.append(float((1/2*sigma_new)*np.trace(Ezzt_n.dot(W_new.T).dot(W_new))))

				metric_new = -sum(a+b+c-d+e)
				print('Maximization number',str(counter),'expectation metric:',metric_new)

				if abs(metric_new-metric)<0.00000001: #10^-8
					print('Converged at ', counter, '!')
					break
				else:
					W = W_new
					sigma = sigma_new
					metric = metric_new
			return W_new

		EMruns=10
		for x in range(EMruns):
			Wfinal+=EM(X)
		Wfinal=Wfinal/EMruns #Running EM multiple times and then keeping the mean Matrix of all the outputs
		U,S,V=np.linalg.svd(Wfinal, full_matrices=False)
		Output=U.T.dot(Data)
		return(Output)

def KPCA(X, gamma=3, dims=1, mode='gaussian'):
	print('Now running Kernel PCA with', mode, 'kernel function...')
	'''
	X is the necessary input. The data.
	gamma will be the user defined value that will be used in the kernel functions. The default is 3.
	dims will be the number of dimensions of the final output (basically the number of components to be picked). The default is 1.
	mode has three options 'gaussian', 'polynomial', 'hyperbolic tangent' which will be the kernel function to be used. The default is gaussian.
	'''

	#First the kernel function picked by the user is defined. Vectors need to be input in np.mat type

	def phi(x1,x2):
		if mode == 'gaussian':
			return (float(np.exp(-gamma*((x1-x2).dot((x1-x2).T))))) #gaussian. (vectors are rather inconvenient in python, so instead of xTx for inner product we need to calculate xxT)
		if mode == 'polynomial':
			return (float((1 + x1.dot(x2.T))**gamma)) #polynomial
		if mode == 'hyperbolic tangent':
			return (float(np.tanh(x1.dot(x2.T) + gamma))) #hyperbolic tangent
	Kernel=[]
	for x in X.T:
		xi=np.mat(x)
		row=[]
		for y in X.T:
			xj=np.mat(y)
			kf=phi(xi,xj)
			row.append(kf)
		Kernel.append(row)
	kernel=np.array(Kernel)

	# Centering the symmetric NxN kernel matrix.
	N = kernel.shape[0]
	one_n = np.ones((N,N)) / N
	kernel = kernel - one_n.dot(kernel) - kernel.dot(one_n) + one_n.dot(kernel).dot(one_n) #centering

	eigVals, eigVecs = linalg.eigh(kernel) #the eigvecs are sorted in ascending eigenvalue order.
	y=eigVecs[:,-dims:].T #user defined dims, since the order is reversed, we pick principal components from the last columns instead of the first
	return (y)


# Copyright 2016 Google Inc. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Parse cloudbuild config files.

"""
import base64
import os

from apitools.base.protorpclite import messages as proto_messages
from apitools.base.py import encoding as apitools_encoding

from googlecloudsdk.api_lib.cloudbuild import cloudbuild_util
from googlecloudsdk.core import exceptions
import yaml
import yaml.parser

# Don't apply camel case to keys for dict or list values with these field names.
# These correspond to map fields in our proto message, where we expect keys to
# be sent exactly as the user typed them, without transformation to camelCase.
_SKIP_CAMEL_CASE = ['secretEnv', 'secret_env', 'substitutions']

class NotFoundException(exceptions.Error):

  def __init__(self, path):
    msg = '{path} could not be found'.format(
        path=path or 'Cloud Build configuration',
    )
    super(NotFoundException, self).__init__(msg)

class FileReadException(exceptions.Error):

  def __init__(self, path):
    msg = '{path} could not be read'.format(
        path=path or 'Cloud Build configuration',
    )
    super(FileReadException, self).__init__(msg)

class ParserError(exceptions.Error):

  def __init__(self, path, msg):
    msg = 'parsing {path}: {msg}'.format(
        path=path or 'Cloud Build configuration',
        msg=msg,
    )
    super(ParserError, self).__init__(msg)

class BadConfigException(exceptions.Error):

  def __init__(self, path, msg):
    msg = '{path}: {msg}'.format(
        path=path or 'Cloud Build configuration',
        msg=msg,
    )
    super(BadConfigException, self).__init__(msg)

def _SnakeToCamelString(field_name):
  """Change a snake_case string into a camelCase string.

  Args:
    field_name: str, the string to be transformed.

  Returns:
    str, the transformed string.
  """
  parts = field_name.split('_')
  if not parts:
    return field_name

  # Handle field_name with leading '_'s by collapsing them into the next part.
  # Legit field names will never look like this, but completeness of the
  # function is important.
  leading_blanks = 0
  for p in parts:
    if not p:
      leading_blanks += 1
    else:
      break
  if leading_blanks:
    parts = parts[leading_blanks:]
    if not parts:
      # If they were all blanks, then we over-counted by one because of split
      # behavior.
      return '_'*(leading_blanks-1)
    parts[0] = '_'*leading_blanks + parts[0]

  return ''.join(parts[:1] + [s.capitalize() for s in parts[1:]])

def _SnakeToCamel(msg):
  """Transform all dict field names that are snake_case to camelCase.

  If a field is in _SKIP_CAMEL_CASE then its value is not further transformed.

  Args:
    msg: dict, list, or other. If 'other', the function returns immediately.

  Returns:
    Same type as message, except all field names except "secrets" that were
    snake_case are now camelCase.
  """
  if isinstance(msg, dict):
    return {
        _SnakeToCamelString(key):
        _SnakeToCamel(val) if key not in _SKIP_CAMEL_CASE else val
        for key, val in msg.iteritems()
    }
  elif isinstance(msg, list):
    return [_SnakeToCamel(elem) for elem in msg]
  else:
    return msg

def _UnpackCheckUnused(obj, msg_type):
  """Stuff a dict into a proto message, and fail if there are unused values.

  Args:
    obj: dict(), The structured data to be reflected into the message type.
    msg_type: type, The proto message type.

  Raises:
    ValueError: If there is an unused value in obj.

  Returns:
    Proto message, The message that was created from obj.
  """
  msg = apitools_encoding.DictToMessage(obj, msg_type)

  def _CheckForUnusedFields(obj):
    """Check for any unused fields in nested messages or lists."""
    if isinstance(obj, proto_messages.Message):
      unused_fields = obj.all_unrecognized_fields()
      if unused_fields:
        if len(unused_fields) > 1:
          # Because this message shows up in a dotted path, use braces.
          # eg .foo.bar.{x,y,z}
          unused_msg = '{%s}' % ','.join(sorted(unused_fields))
        else:
          # For single items, omit the braces.
          # eg .foo.bar.x
          unused_msg = unused_fields[0]
        raise ValueError('.%s: unused' % unused_msg)
      for used_field in obj.all_fields():
        try:
          field = getattr(obj, used_field.name)
          _CheckForUnusedFields(field)
        except ValueError as e:
          raise ValueError('.%s%s' % (used_field.name, e))
    if isinstance(obj, list):
      for i, item in enumerate(obj):
        try:
          _CheckForUnusedFields(item)
        except ValueError as e:
          raise ValueError('[%d]%s' % (i, e))

  _CheckForUnusedFields(msg)

  return msg

def LoadCloudbuildConfigFromStream(stream, messages, params=None,
                                   path=None):
  """Load a cloudbuild config file into a Build message.

  Args:
    stream: file-like object containing the JSON or YAML data to be decoded
    messages: module, The messages module that has a Build type.
    params: dict, parameters to substitute into the Build spec.
    path: str or None. Optional path to be used in error messages.

  Raises:
    NotFoundException: If the file does not exist.
    ParserError: If there was a problem parsing the file.
    BadConfigException: If the config file has illegal values.

  Returns:
    Build message, The build that got decoded.
  """
  # Turn the data into a dict
  try:
    structured_data = yaml.safe_load(stream)
    if not isinstance(structured_data, dict):
      raise ParserError(path, 'Could not parse into a message.')
  except yaml.parser.ParserError as pe:
    raise ParserError(path, pe)

  # Transform snake_case into camelCase.
  structured_data = _SnakeToCamel(structured_data)

  # Then, turn the dict into a proto message.
  try:
    build = _UnpackCheckUnused(structured_data, messages.Build)
  except ValueError as e:
    raise BadConfigException(path, '%s' % e)

  subst = structured_data.get('substitutions', {})
  if params:
    subst.update(params)
  build.substitutions = cloudbuild_util.EncodeSubstitutions(subst, messages)

  # Re-base64-encode secrets[].secretEnv values, which apitools' DictToMessage
  # "helpfully" base64-decodes since it can tell it's a bytes field. We need to
  # send a base64-encoded string in the JSON request, not raw bytes.
  for s in build.secrets:
    for i in s.secretEnv.additionalProperties:
      i.value = base64.b64encode(i.value)

  # Some problems can be caught before talking to the cloudbuild service.
  if build.source:
    raise BadConfigException(path, 'config cannot specify source')
  if not build.steps:
    raise BadConfigException(path, 'config must list at least one step')

  return build

def LoadCloudbuildConfigFromPath(path, messages, params=None):
  """Load a cloudbuild config file into a Build message.

  Args:
    path: str. Path to the JSON or YAML data to be decoded.
    messages: module, The messages module that has a Build type.
    params: dict, parameters to substitute into a templated Build spec.

  Raises:
    NotFoundException: If the file does not exist.
    ParserError: If there was a problem parsing the file.
    BadConfigException: If the config file has illegal values.

  Returns:
    Build message, The build that got decoded.
  """
  if not os.path.exists(path):
    raise NotFoundException(path)

  try:
    with open(path) as f:
      return LoadCloudbuildConfigFromStream(f, messages, params, path=path)
  except EnvironmentError:
    # EnvironmentError is parent of IOError, OSError and WindowsError.
    # Raised when file does not exist or can't be opened/read.
    raise FileReadException(path)


import logging
from datetime import datetime
from threading import Thread, Timer
from time import sleep

from .constants import DPG_COMMAND_HANDLE, REFERENCE_OUTPUT_HANDLE, \
    MOVE_TO_HANDLE
from .connection import BTLEConnection
from .desk_position import DeskPosition
from .dpg_command import DPGCommand, DeskOffsetCommand, MemorySetting2Command
from .height_speed import HeightSpeed

_LOGGER = logging.getLogger(__name__)

NAME_HANDLE = 0x0003

PROP_GET_CAPABILITIES = 0x80
PROP_DESK_OFFSET = 0x81
PROP_USER_ID = 0x86
PROP_MEMORY_POSITION_1 = 0x0089
PROP_MEMORY_POSITION_2 = 0x008a

MOVE_TIMER_TIMEOUT = 30
MIN_SEC_BETWEEN_QUERIES = 0.15

HEIGHT_MIN = 0
HEIGHT_MAX = 64

class DPGCommandReadError(Exception):
    pass

class WrongFavoriteNumber(Exception):
    pass

class LinakDesk:

    def __init__(self, bdaddr):
        self._handlers = {
            REFERENCE_OUTPUT_HANDLE: self._handle_reference_notification,
            DPG_COMMAND_HANDLE: self._handle_dpg_notification,
        }

        self._bdaddr = bdaddr
        self._conn = BTLEConnection(bdaddr, self._handlers)

        self._name = None
        self._desk_offset = None
        self._fav_position_1 = None
        self._fav_position_2 = None
        self._height_speed = None
        self._last_height_query = None

        self._target = None
        self._running = False
        self._manual_height_change = False
        self._stop_timer = None

    @property
    def name(self):
        return self._wait_for_variable('_name')

    @property
    def desk_offset(self):
        return self._wait_for_variable('_desk_offset')

    @property
    def favorite_position_1(self):
        return self._wait_for_variable('_fav_position_1')

    @property
    def favorite_position_2(self):
        return self._wait_for_variable('_fav_position_2')

    @property
    def current_height(self):
        if not self._running:
            self._query_height_speed()
        return self.height_speed.height

    @property
    def current_height_with_offset(self):
        return self._with_desk_offset(self.height_speed.height)

    @property
    def height_speed(self):
        return self._wait_for_variable('_height_speed')

    @property
    def is_running(self):
        return self._running

    @property
    def target_height(self):
        if not self._running:
            return None, None

        direction_up = self._target.raw > self.current_height.raw

        if not self._manual_height_change:
            return self._target.cm, direction_up
        else:
            # height is being changed manually, so no target height
            return None, direction_up

    def _query_initial_data(self):
        with self._conn as conn:
            self._name = conn.read_characteristic(NAME_HANDLE)
            conn.dpg_command(PROP_USER_ID)
            conn.dpg_command(PROP_GET_CAPABILITIES)

    def _query_desk_offset(self):
        with self._conn as conn:
            conn.dpg_command(PROP_DESK_OFFSET)

    def _query_memory_positions(self):
        with self._conn as conn:
            self._fav_position_1 = None
            self._fav_position_2 = None

            conn.dpg_command(PROP_MEMORY_POSITION_1)
            self._wait_for_variable('_fav_position_1')

            conn.dpg_command(PROP_MEMORY_POSITION_2)
            self._wait_for_variable('_fav_position_2')

    def _query_height_speed(self):
        if self._last_height_query is not None:
            delta = (datetime.now() - self._last_height_query).total_seconds()
            if delta < MIN_SEC_BETWEEN_QUERIES:
                return

        with self._conn as conn:
            self._height_speed = HeightSpeed.from_bytes(
                conn.read_characteristic(REFERENCE_OUTPUT_HANDLE))
        self._last_height_query = datetime.now()

    def init(self):
        _LOGGER.debug("Querying the device..")

        """ We need to query for name before doing anything, without it device doesnt respond """
        self._query_initial_data()
        self._query_desk_offset()
        self._query_memory_positions()
        self._query_height_speed()

    def __str__(self):
        return "[%s] Desk offset: %s, name: %s\nFav position1: %s, Fav position 2: %s Height with offset: %s" % (
            self._bdaddr,
            self.desk_offset.human_cm,
            self.name,
            self._with_desk_offset(self.favorite_position_1).human_cm,
            self._with_desk_offset(self.favorite_position_2).human_cm,
            self._with_desk_offset(self.height_speed.height).human_cm,
        )

    def __repr__(self):
        return self.__str__()

    def move_to_cm(self, cm):
        calculated_raw = DeskPosition.raw_from_cm(cm - self._desk_offset.cm)
        self._move_to_raw(calculated_raw)

    def move_down(self):
        self._manual_height_change = True
        self.move_to_cm(self._desk_offset.cm + HEIGHT_MIN)

    def move_up(self):
        self._manual_height_change = True
        self.move_to_cm(self._desk_offset.cm + HEIGHT_MAX)

    def stop_movement(self):
        _LOGGER.debug("Move stopped")
        # send stop move
        self._running = False
        self._manual_height_change = False
        self._stop_timer.cancel()

    def move_to_fav(self, fav):
        if fav == 1:
            raw = self.favorite_position_1.raw
        elif fav == 2:
            raw = self.favorite_position_2.raw
        else:
            raise DPGCommandReadError('Favorite with position: %d does not exists' % fav)

        self._move_to_raw(raw)

    def _wait_for_variable(self, var_name):
        value = getattr(self, var_name)
        if value is not None:
            return value

        for _ in range(0, 100):
            value = getattr(self, var_name)

            if value is not None:
                return value

            sleep(0.2)

        raise DPGCommandReadError('Cannot fetch value for %s' % var_name)

    def _with_desk_offset(self, value):
        return DeskPosition(value.raw + self.desk_offset.raw)

    def _handle_dpg_notification(self, data):
        """Handle Callback from a Bluetooth (GATT) request."""
        _LOGGER.debug("Received notification from the device..")

        if data[0] != 0x1:
            raise DPGCommandReadError('DPG_Control packets needs to have 0x01 in first byte')

        command = DPGCommand.build_command(data)
        _LOGGER.debug("Received %s (%s)", command.__class__.__name__, command.decoded_value)

        if command.__class__ == DeskOffsetCommand:
            self._desk_offset = command.offset
        elif command.__class__ == MemorySetting2Command:
            # DPG1M replies to queries for both memory positions with the same
            # command type (0x07)
            if self._fav_position_1 is None:
                self._fav_position_1 = command.offset
            else:
                self._fav_position_2 = command.offset

    def _handle_reference_notification(self, data):
        self._height_speed = HeightSpeed.from_bytes(data)

        _LOGGER.debug("Current relative height: %s, speed: %f", self._height_speed.height.human_cm, self._height_speed.speed.parsed)

    def _send_move_to(self):
        with self._conn as conn:
            _LOGGER.debug("Sending move to: %s", self._target.human_cm)
            conn.make_request(MOVE_TO_HANDLE, self._target.bytes, timeout=None)

    def _move_to_raw(self, raw_value):
        if self._running:
            self.stop_movement()

        current_raw_height = self.height_speed.height.raw
        if abs(raw_value - current_raw_height) < 10:
            _LOGGER.debug("Move not possible, current raw height: %d", current_raw_height)
            return

        self._target = DeskPosition(raw_value)
        self._running = True

        self._stop_timer = Timer(MOVE_TIMER_TIMEOUT, self.stop_movement)
        self._stop_timer.start()

        _LOGGER.debug("Start move to: %s", self._target.human_cm)

        Thread(target=self._process_movement).start()

    def _process_movement(self):
        while self._running:
            self._send_move_to()
            sleep(0.2)

            self._query_height_speed()
            if self.height_speed.speed.parsed < 0.001:
                self.stop_movement()


import argparse

import numpy as np
import tables

def main():
    run(**parse_args())

def parse_args():
    parser = argparse.ArgumentParser()
    parser.add_argument("filename", type=str)
    parser.add_argument("path", type=str)
    return vars(parser.parse_args())

def run(filename, path):
    with tables.open_file(filename) as store:
        node = store.root
        for name in path.split("/"):
            node = node[name]

        print("Node:")
        print(node)

        print("Data:")
        if isinstance(node, tables.Leaf):
            print("")
            print(node[:])

if __name__ == "__main__":
    main()


# Copyright 2017 reinforce.io. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================

"""
Test for examples from the reinforce.io website, blogposts and other examples.
"""

from __future__ import absolute_import
from __future__ import print_function
from __future__ import division

import unittest

class TestTutorialCode(unittest.TestCase):
    """
    Validation of random code snippets as to be notified when old blog posts need to be changed.
    """

    class MyClient(object):

        def __init__(self, *args, **kwargs):
            pass

        def get_state(self):
            import numpy as np
            return np.random.rand(10)

        def execute(self, action):
            pass

    def test_reinforceio_homepage(self):
        """
        Code example from the homepage and README.md.
        """

        from tensorforce import Configuration
        from tensorforce.agents import TRPOAgent
        from tensorforce.core.networks import layered_network_builder

        config = Configuration(
            batch_size=100,
            states=dict(shape=(10,), type='float'),
            actions=dict(continuous=False, num_actions=2),
            network=layered_network_builder([dict(type='dense', size=50), dict(type='dense', size=50)])
        )

        # Create a Trust Region Policy Optimization agent
        agent = TRPOAgent(config=config)

        # Get new data from somewhere, e.g. a client to a web app
        client = TestTutorialCode.MyClient('http://127.0.0.1', 8080)

        # Poll new state from client
        state = client.get_state()

        # Get prediction from agent, execute
        action = agent.act(state=state)
        reward = client.execute(action)

        # Add experience, agent automatically updates model according to batch size
        agent.observe(reward=reward, terminal=False)

    def test_blogpost_introduction(self):
        """
        Test of introduction blog post examples.
        """
        import tensorflow as tf
        import numpy as np

        ### DQN agent example

        from tensorforce import Configuration
        from tensorforce.agents import DQNAgent
        from tensorforce.core.networks import layered_network_builder

        # Define a network builder from an ordered list of layers
        layers = [dict(type='dense', size=32),
                  dict(type='dense', size=32)]
        network = layered_network_builder(layers_config=layers)

        # Define a state
        states = dict(shape=(10,), type='float')

        # Define an action (models internally assert whether
        # they support continuous and/or discrete control)
        actions = dict(continuous=False, num_actions=5)

        # The agent is configured with a single configuration object
        agent_config = Configuration(
            batch_size=8,
            learning_rate=0.001,
            memory_capacity=800,
            first_update=80,
            repeat_update=4,
            target_update_frequency=20,
            states=states,
            actions=actions,
            network=network
        )
        agent = DQNAgent(config=agent_config)

        ### Code block: multiple states

        states = dict(
            image=dict(shape=(64, 64, 3), type='float'),
            caption=dict(shape=(20,), type='int')
        )

        agent_config.states = states
        # DQN does not support multiple states. Omit test for now.
        # agent = DQNAgent(config=agent_config)

        ### Code block: DQN observer function

        def observe(self, reward, terminal):
            super(DQNAgent, self).observe(reward, terminal)
            if self.timestep >= self.first_update \
                    and self.timestep % self.target_update_frequency == 0:
                self.model.update_target()

        ### Code block: Network config JSON

        network_json = """
        [
            {
                "type": "conv2d",
                "size": 32,
                "window": 8,
                "stride": 4
            },
            {
                "type": "conv2d",
                "size": 64,
                "window": 4,
                "stride": 2
            },
            {
                "type": "flatten"
            },
            {
                "type": "dense",
                "size": 512
            }
        ]
        """

        ### Test json

        import json
        network_config = json.loads(network_json)
        network = layered_network_builder(network_config)

        ### Test from_json import

        from tensorforce.core.networks import from_json

        ### Code block: Modified dense layer

        modified_dense = """
        [
            {
                "type": "dense",
                "size": 64,
                "bias": false,
                "activation": "selu",
                "l2_regularization": 0.001
            }
        ]
        """

        ### Test json

        network_config = json.loads(network_json)
        network = layered_network_builder(network_config)

        ### Code block: Own layer type

        def batch_normalization(x, variance_epsilon=1e-6, **kwargs):
            mean, variance = tf.nn.moments(x, axes=tuple(range(x.shape.ndims - 1)))
            x = tf.nn.batch_normalization(x, mean=mean, variance=variance, offset=None, scale=None,
                                          variance_epsilon=variance_epsilon)
            return x

        ### Test own layer

        network_config = [{"type": batch_normalization,
                           "variance_epsilon": 1e-9}]
        network = layered_network_builder(network_config)
        agent_config.network=network
        agent_config.states = dict(shape=(10,), type='float')
        agent = DQNAgent(config=agent_config)

        ### Code block: Own network builder

        def network_builder(inputs, **kwargs):
            image = inputs['image']  # 64x64x3-dim, float
            caption = inputs['caption']  # 20-dim, int

            with tf.variable_scope('cnn'):
                weights = tf.Variable(tf.random_normal(shape=(3, 3, 3, 16), stddev=0.01))
                image = tf.nn.conv2d(image, filter=weights, strides=(1, 1, 1, 1), padding='SAME')
                image = tf.nn.relu(image)
                image = tf.nn.max_pool(image, ksize=(1, 2, 2, 1), strides=(1, 2, 2, 1), padding='SAME')

                weights = tf.Variable(tf.random_normal(shape=(3, 3, 16, 32), stddev=0.01))
                image = tf.nn.conv2d(image, filter=weights, strides=(1, 1, 1, 1), padding='SAME')
                image = tf.nn.relu(image)
                image = tf.nn.max_pool(image, ksize=(1, 2, 2, 1), strides=(1, 2, 2, 1), padding='SAME')

                image = tf.reshape(image, shape=(-1, 16 * 16, 32))
                image = tf.reduce_mean(image, axis=1)

            with tf.variable_scope('lstm'):
                weights = tf.Variable(tf.random_normal(shape=(30, 32), stddev=0.01))
                caption = tf.nn.embedding_lookup(params=weights, ids=caption)
                lstm = tf.contrib.rnn.LSTMCell(num_units=32)
                caption, _ = tf.nn.dynamic_rnn(cell=lstm, inputs=caption, dtype=tf.float32)
                caption = tf.reduce_mean(caption, axis=1)

            return tf.multiply(image, caption)

        ### Test own network builder

        agent_config.states = dict(
            image=dict(shape=(64, 64, 3), type='float'),
            caption=dict(shape=(20,), type='int')
        )
        agent_config.network=network_builder
        agent = DQNAgent(config=agent_config)

        ### Code block: LSTM function

        def lstm(x, **kwargs):
            size = x.get_shape()[1].value
            internal_input = tf.placeholder(dtype=tf.float32, shape=(None, 2, size))
            lstm = tf.contrib.rnn.LSTMCell(num_units=size)
            state = tf.contrib.rnn.LSTMStateTuple(internal_input[:, 0, :],
                                                  internal_input[:, 1, :])
            x, state = lstm(inputs=x, state=state)
            internal_output = tf.stack(values=(state.c, state.h), axis=1)
            internal_init = np.zeros(shape=(2, size))
            return x, [internal_input], [internal_output], [internal_init]

        ### Test LSTM

        network_config = [{"type": lstm}]
        network = layered_network_builder(network_config)
        agent_config.network=network
        agent_config.states = dict(shape=(10,), type='float')
        agent = DQNAgent(config=agent_config)

        ### Preprocessing configuration

        agent_config.preprocessing = [
            dict(
                type='image_resize',
                width=84,
                height=84
            ),
            dict(
                type='grayscale'
            ),
            dict(
                type='center'
            ),
            dict(
                type='sequence',
                length=4
            )
        ]

        ### Test preprocessing configuration

        agent = DQNAgent(config=agent_config)

        #agent_config.actions = dict(continuous=False, num_actions=5)

        ### Code block: Continuous action exploration

        agent_config.exploration = dict(
            type='OrnsteinUhlenbeckProcess',
            sigma=0.1,
            mu=0,
            theta=0.1
        )

        ### Test continuous action exploration

        agent = DQNAgent(config=agent_config)

        ### Code block: Discrete action exploration

        agent_config.exploration = dict(
            type='EpsilonDecay',
            epsilon=1,
            epsilon_final=0.01,
            epsilon_timesteps=1e6
        )

        ### Test discrete action exploration

        agent = DQNAgent(config=agent_config)

    def test_blogpost_introduction_runner(self):
        from tensorforce.config import Configuration
        from tensorforce.core.networks import layered_network_builder
        from tensorforce.environments.minimal_test import MinimalTest
        from tensorforce.agents import DQNAgent
        from tensorforce.execution import Runner

        environment = MinimalTest(definition=False)

        network_config = [
            dict(type='dense', size=32)
        ]
        agent_config = Configuration(
            batch_size=8,
            learning_rate=0.001,
            memory_capacity=800,
            first_update=80,
            repeat_update=4,
            target_update_frequency=20,
            states=environment.states,
            actions=environment.actions,
            network=layered_network_builder(network_config)
        )

        agent = DQNAgent(config=agent_config)
        runner = Runner(agent=agent, environment=environment)

        def episode_finished(runner):
            if runner.episode % 100 == 0:
                print(sum(runner.episode_rewards[-100:]) / 100)
            return runner.episode < 100 \
                   or not all(reward >= 1.0 for reward in runner.episode_rewards[-100:])

        # runner.run(episodes=1000, episode_finished=episode_finished)
        runner.run(episodes=10, episode_finished=episode_finished)  # Only 10 episodes for this test

        ### Code block: next

        # max_episodes = 1000
        max_episodes = 10  # Only 10 episodes for this test
        max_timesteps = 2000

        episode = 0
        episode_rewards = list()

        while True:
            state = environment.reset()
            agent.reset()

            timestep = 0
            episode_reward = 0
            while True:
                action = agent.act(state=state)
                state, reward, terminal = environment.execute(action=action)
                agent.observe(reward=reward, terminal=terminal)

                timestep += 1
                episode_reward += reward

                if terminal or timestep == max_timesteps:
                    break

            episode += 1
            episode_rewards.append(episode_reward)

            if all(reward >= 1.0 for reward in episode_rewards[-100:]) \
                    or episode == max_episodes:
                break

{'Configuration': 'tensorforce.Configuration', 'TRPOAgent': 'tensorforce.agents.TRPOAgent', 'layered_network_builder': 'tensorforce.core.networks.layered_network_builder'}.MyClient


#!/usr/bin/env python3

""" This modules provides utility classes and functions """

import glob
import importlib
import json
import os
import pwd
import re
import shlex
import shutil
import subprocess
import sys

from pathlib import Path
from os.path import abspath, basename, dirname, isfile, join, splitext
from subprocess import CompletedProcess, DEVNULL, run
from typing import Callable, Optional

__version__ = "1.2.0"

__repo_dir__ = Path(__file__).parents[2]

def load_categories() -> None:
    category_dir = join(dirname(abspath(__file__)), "categories")
    sys.path.append(category_dir)

    for file in glob.iglob(join(category_dir, "**/*.py"), recursive=True):
        if isfile(file):
            importlib.import_module(splitext(basename(file))[0])

def require_repo_dir(name) -> Path:
    repo_dir = __repo_dir__ / name

    if not repo_dir.is_dir():
        raise NotADirectoryError("Cannot find directory '%s' in repository"
                                 % name)

    return repo_dir

def is_root() -> bool:
    return os.getuid() == 0

def require_root(func) -> Callable:
    """ Function decorator for functions that require root privileges. """

    def new_function(*args, **kwargs):
        if not is_root():
            raise PermissionError("Cannot perform this operation: "
                                  "Missing root privileges")
        return func(*args, **kwargs)

    return new_function

def get_dist() -> Optional[str]:
    """ Returns the name of the linux distribution on this system or None """
    with open("/etc/os-release", "r") as file:
        for line in file.read().splitlines():
            if re.match("^ID=", line):
                return line.lstrip("ID=")

    return None

def parse_json_descriptor(path) -> dict:
    """ Parses the specified json file and returns it as dict """
    with open(str(path), "r", encoding="utf-8") as file:
        json_file = json.load(file)

    return json_file

def remove_kv_pair(collection, option) -> bool:
    """ Removes the first option value pair from the collection. """
    index = -1
    if option in collection:
        index = collection.index(option)
        del collection[index:index + 2]

    return index != -1

def remove_user_options(argv=None) -> list:
    """ Removes all -u and --user option value pairs from argv or sys.argv. """
    if argv is None:
        argv = sys.argv
    argv = argv.copy()

    while remove_kv_pair(argv, "-u") or remove_kv_pair(argv, "--user"):
        pass

    return argv

def start_subprocess(args, user) -> subprocess.Popen:
    """ Executes the given arguments as the specified user. """
    pw_record = pwd.getpwnam(user)
    user_name = pw_record.pw_name
    user_home = pw_record.pw_dir
    uid = pw_record.pw_uid
    gid = pw_record.pw_gid

    cwd = os.getcwd()

    env = os.environ.copy()
    env["HOME"] = user_home
    env["LOGNAME"] = user_name
    env["PWD"] = cwd
    env["USER"] = user_name

    process = subprocess.Popen(
        args, preexec_fn=demote(uid, gid), cwd=cwd, env=env
    )

    return process

def demote(uid, gid) -> Callable[[], None]:
    """ Returns a functions that sets the uid and gid for the current user. """

    def result():
        try:
            os.setgid(gid)
            os.setuid(uid)
        except PermissionError:
            sys.exit(2)

    return result

class SetupUtils(object):
    def __init__(self, namespace=None):
        args_dict = vars(namespace) if namespace is not None else {}

        self.link = args_dict.get("link", True)

        dst_handling = args_dict.get("dst_handling", "keep")
        self.keep = (dst_handling == "keep")
        self.backup = (dst_handling == "backup")
        self.delete = (dst_handling == "delete")

        self.verbose = args_dict.get("verbose", False)
        self.quiet = args_dict.get("quiet", False)
        self.confirm = args_dict.get("confirm", True)  # TODO implement

        self.delete_backups = args_dict.get("delete_backups", False)

    @staticmethod
    def _delete_file(file) -> None:
        if file.is_file() or file.is_symlink():
            file.unlink()
        elif file.is_dir():
            shutil.rmtree(str(file))

    def symlink(self, src, dst) -> None:
        """ Create a symlink called dst pointing to src. """
        if not self.link:
            return

        if dst.exists() or dst.is_symlink():
            if not self.keep:
                raise FileExistsError("Cannot create link '%s': File exists"
                                      % str(dst))
            else:
                return

        if not src.exists() and not src.is_symlink():
            raise FileNotFoundError("Cannot link to '%s': No such file exists"
                                    % str(src))

        self.print_create_symlink(dst, src)
        dst.parent.mkdir(parents=True, exist_ok=True)
        dst.symlink_to(src)

    def backup_file(self, src) -> None:
        if not self.backup:
            return

        if not src.exists() and not src.is_symlink():
            return

        dst = src.with_suffix(".old")
        self.print_move(src, dst)
        os.rename(str(src), str(dst))

    def delete_file(self, file) -> None:
        if not self.delete:
            return

        if not file.exists() and not file.is_symlink():
            return

        self.print_delete(file)
        self._delete_file(file)

    def delete_backup(self, file) -> None:
        if not self.delete_backups:
            return

        backup = file.with_suffix(".old")

        if not backup.exists() and not backup.is_symlink():
            return

        self.print_delete(backup)
        self._delete_file(backup)

    def run(self, args) -> CompletedProcess:
        kwargs = dict(stdout=None if self.verbose else DEVNULL,
                      stderr=DEVNULL if self.quiet else None, check=False)

        return run(args, **kwargs)

    def clone_repo(self, url, path, name=None) -> None:
        if name is None:
            name = path.name

        path = path.expanduser()

        self.error("Installing %s to '%s'..." % (name, str(path)))

        if path.exists():
            if path.joinpath(".git").exists():
                self.error("%s is already installed, updating ..."
                        % name, prefix="warning:")

                args = ["git", "-C", str(path), "pull", "-v"]
                if not self.verbose:
                    args.remove("-v")

                process = self.run(args)

                if process.returncode != 0:
                    self.error("Failed to update %s: Exited with exit code %s"
                            % (name, process.returncode), prefix="error:")
                return

            self.error("'%s' already exists, but is no git repo" % path,
                    prefix="error:")
            if not self.confirm("Clone into the existing directory?", False):
                return self.error("Skipping installation of %s" % name,
                        prefix="warning:")

        path.mkdir(parents=True, exist_ok=True)

        args = ["git", "clone", "-v", str(url), str(path)]
        if not self.verbose:
            args.remove("-v")

        process = self.run(args)

        if process.returncode != 0:
            self.error("Failed to install %s: Exited with code %s"
                    % (name, process.returncode), prefix="error:")

    def try_execute(self, func) -> None:
        try:
            func()
        except (OSError, PermissionError) as e:
            self.error(str(e), prefix="error:")

    def install_packages(self, dist, *packages) -> None:
        if dist == "arch":
            command = "pacman -S --needed --noconfirm %s"
        elif dist == "debian":
            command = "apt --assume-yes install %s"
        else:
            raise OSError("Cannot install packages: Unknown or unsupported "
                          "linux distribution '%s'" % dist)

        self._install_packages(command, *packages)

    def install_pip_packages(self, *packages) -> None:
        self._install_packages("pip install %s", *packages)

    def install_npm_packages(self, *packages) -> None:
        self._install_packages("npm install -g %s", *packages)

    def install_gem_packages(self, *packages) -> None:
        self._install_packages("gem install %s --no-user-install", *packages)

    def _install_packages(self, command, *packages) -> None:
        if len(packages) == 0:
            return

        process = self.run(shlex.split(command % " ".join(packages)))

        if process.returncode != 0:
            self.error("Failed to install packages '%s': Exited with code %s"
                    % (", ".join(packages), process.returncode),
                    prefix="error:")

    def print(self, prefix=None, *args, **kwargs) -> None:
        prefix = "setup.py:" if prefix is None else prefix
        if self.verbose:
            print(prefix, *args, **kwargs)

    def error(self, *args, prefix=None, **kwargs) -> None:
        prefix = "setup.py:" if prefix is None else prefix
        if not self.quiet:
            print(prefix, *args, file=sys.stderr, **kwargs)

    def confirm(self, msg, default=True) -> bool:
        """ Waits for user input to confirm or uses the default. """
        if self.quiet or not self.confirm:
            return default

        self.error(msg, "[Y/n]" if default else "[y/N]", end=" ")

        user_input = input()
        choices = {'y': True, 'Y': True, 'n': False, 'N': False, None: default,
                   '': default}

        return choices.get(user_input, not default)

    def print_create_symlink(self, src, dst) -> None:
        self.print("Creating link: '%s' -> '%s'" % (str(src), str(dst)))

    def print_delete(self, file) -> None:
        self.print("Deleting file: '%s'" % str(file))

    def print_move(self, src, dst) -> None:
        self.print("Moving file: '%s' -> '%s'" % (str(src), str(dst)))

class FileMapping(object):
    """
    Represents a mapping between a file in this repository and a file
    on this system.
    """

    utils = SetupUtils()

    def __init__(self, src, dst, root=False, distribution=None):
        """
        :param src: the absolute path to the file in the repository
        :param dst: the absolute path to the file on the system
        :param root: whether root privileges are required for the file
        :param distribution: the (list of) linux distribution(s) the
                             file is used on
        """
        self.src = src
        self.dst = dst
        self.root = root
        self.distributions = list()

        if type(distribution) is str:
            self.distributions.append(distribution)
        elif type(distribution) is list:
            self.distributions.extend(distribution)
        elif distribution is not None:
            raise ValueError("Distribution must be either string or list")

    def __repr__(self):
        return "FileMapping(src='%s', dst='%s')" % (self.src, self.dst)

    @classmethod
    def require_utils(cls, utils) -> SetupUtils:
        return utils if utils is not None else cls.utils

    def is_privileged(self) -> bool:
        return is_root() >= self.root

    def is_distribution(self) -> bool:
        return get_dist() in self.distributions or not self.distributions

    def can_setup(self) -> bool:
        return self.is_privileged() and self.is_distribution()

    def with_suffix(self, suffix):
        dst = self.dst + suffix
        return FileMapping(self.src, dst, self.root, self.distributions)

    def link(self, utils=None) -> None:
        if self.can_setup():
            self.require_utils(utils).symlink(self.src, self.dst)

    def delete_dst(self, utils=None) -> None:
        if self.can_setup():
            self.require_utils(utils).delete_file(self.dst)

    def delete_backup(self, utils=None) -> None:
        if self.can_setup():
            self.require_utils(utils).delete_backup(self.dst)

    def backup_dst(self, utils=None) -> None:
        if self.can_setup():
            self.require_utils(utils).backup_file(self.dst)


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Sat Dec 14 01:00:04 2019

@author: peter
"""

import seaborn as sns
import pandas as pd
import matplotlib.pyplot as plt
from matplotlib.lines import Line2D
import numpy as np
import math

sns.set(context='paper')

def GFID(myDict):
    """Get First in Dictionary

    A function to strip the first and assumed only eliment out of a
    dictionary.

    Args:
       myDict (Dict):  Dictioary wraped around single member.

    Returns:
       what ever was in the dictionary.
    """
    return (myDict[next(iter(myDict))])

def unevenSubplots(number, col, sharex=True, sharey=False,
                   figsize=(12,10)):
    fig = plt.figure(figsize=figsize)
    rows = number//col-(number%col==0)+1
    axs = []
    for i in range(number):
        if sharex and i>0 and (not sharey):
            axs.append(fig.add_subplot(rows, col, i+1, sharex=axs[0]))
        elif (not sharex) and i>0 and sharey:
            axs.append(fig.add_subplot(rows, col, i+1, sharey=axs[0]))
        elif sharex and i>0 and sharey:
            axs.append(fig.add_subplot(rows, col, i+1, sharex=axs[0],
                                       sharey=axs[0]))
        else:
            axs.append(fig.add_subplot(rows, col, i+1))
    if sharey:
        for i, ax in enumerate(axs):
            if i%col>0:
                for label in ax.get_yticklabels():
                    label.set_visible(False)
                ax.yaxis.offsetText.set_visible(False)
                ax.yaxis.label.set_visible(False)
    if sharex:
        for i, ax in enumerate(axs):
            if i+col < number:
                for label in ax.get_xticklabels():
                    label.set_visible(False)
                ax.xaxis.offsetText.set_visible(False)
                ax.xaxis.label.set_visible(False)
    return fig, axs

def plotPanels(df, xVal, yVal, panelVal, hueVal = None, save=None):
    panVals = pd.unique(df[panelVal])
    if hueVal is not None:
        hueVals = pd.unique(df[hueVal])
    else:
        hueVals = [None]
    rows = math.ceil(math.sqrt(3*len(panVals)/5))
    cols = len(panVals)//rows+(len(panVals)%rows>0)
    fig, axs = unevenSubplots(len(panVals), cols)
    for pan, ax in zip(panVals, axs):
        ax.set_title(pan)
        for hueIndex, theHueVal in enumerate(hueVals):
            subDF = df[df[panelVal]==pan]
            if hueVal is not None:
                subDF = subDF[subDF[hueVal]==theHueVal]
            for _, g in subDF.groupby((subDF[xVal].diff() < 0).cumsum()):
                ax.plot(g[xVal],g[yVal],"C"+str(hueIndex))
    fig.tight_layout()
    if save is not None:
        fig.savefig(save)

def trim_axs(axs, N):
    """removes unwanted subplots from subplots function

    Convenience function used to remove unnneeded axises generated
    by the subplots function and return them in a flat structure

    Args:
       axs (array of Axes objects):  output axises from subplots
       N (int): number of subplots you actualy want.

    Returns:
       array of Axes objects: your remaining axises
    """
    axs = axs.flat
    for ax in axs[N:]:
        ax.remove()
    return axs[:N]

def getTCSelectionMaxes(TCList,selectionList=None,varSelection=None,
                       valRemove=[]):
    """Get TimeCourse maximum values by variable

    Gets the maximum value for each variable over all TimeCouses and
    all their time points. Useful in ordering and grouping multi plots.

    Args:
       TCList (list):  A list of TimeCourse Dataframes

    Kwargs:
       selectionList (list of ints): an optional list of the TimeCourses
           (by their indexing in the TCList) to be included in the
           calculation.
       varSelection (list of str): an optional list of variables from the
           TimeCourses we want returned in the results.
       valRemove (list of float): an optional list of values that are to be
           excluded from the list. A quick and dirty way to exclude
           variables that have all been set to some pre determined value.

    Returns:
       serise: the maximum value found for each variable
    """
    if selectionList is None:
        maxes = [course.max() for course in TCList]
    else:
        maxes = [TCList[selection].max() for selection in selectionList]
    maxes = pd.DataFrame(maxes).max()
    maxes = maxes.drop(labels=['Time'])
    if varSelection is not None:
        maxes = maxes.filter(items=varSelection)
    maxes = maxes.sort_values(ascending=False)
    for value in valRemove:
        maxes = maxes[maxes!=value]
    return maxes

def breakSeriesByScale(mySerise,relativeScaleBreakPoint=0.1,
                       maxRunLength=6):
    """breaks Series into subparts of similar size

    Takes a serise and breaks it into parts of comparable value size.
    Sub parts are returned sorted internaly and relative to each other in
    decending order. Useful for ordering and grouping plots with very
    difrent value ranges.

    Args:
       mySerise (serise):  A serise of floats to break into parts

    Kwargs:
       relativeScaleBreakPoint (float): Specifies the smallest (by scale)
           drop in value that should force an entry into a new subpart.
       maxRunLength (int): Max size of any subpart.

    Returns:
       list of list of str: the indexes of the serise are returned broken
       into subparts (the inner list) acording the the size of their
       asociated values.
    """
    outerList=[]
    innerList=[]
    lastVal=None
    for index, value in mySerise.sort_values(ascending=False).items():
        if lastVal is None:
            innerList.append(index)
            lastVal=value
            continue
        if (lastVal*relativeScaleBreakPoint>value or
            len(innerList)==maxRunLength):
            outerList.append(innerList)
            innerList=[]
        else:
            innerList.append(index)
        lastVal=value
    if len(innerList)>0:
        outerList.append(innerList)
    outerList = [myList for myList in outerList if len(myList)>=1]
    return outerList

class profileLikelyhoodVisualisor:
    def __init__(self,profLike):
        idenDict = profLike[0]
        refDict = profLike[1]
        self.scaledDict = {}
        self.fitDict = {}
        self.aggDF = []
        for idenPage in idenDict.keys():
            temp = idenDict[idenPage].copy()
            temp["variable"] = idenPage
            temp = temp.sort_values(by=["RSS"])
            temp["LEVEL"] = temp.groupby(idenPage).cumcount()
            temp = temp.sort_values(by=["LEVEL",idenPage])
            temp = temp.reset_index(drop=True)
            self.aggDF.append(temp)
        self.aggDF = pd.concat(self.aggDF,ignore_index=True)
        for variable, origVal in refDict.items():
            if origVal!=0 and variable in self.aggDF.columns:
                self.aggDF[variable] = self.aggDF[variable]/origVal
        for idenPage in idenDict.keys():
            self.scaledDict[idenPage] = idenDict[idenPage].copy()
            temp = self.scaledDict[idenPage][idenPage]==refDict[idenPage]
            refRow = self.scaledDict[idenPage][temp].squeeze()
            if not isinstance(refRow,pd.Series):
                try:
                    refRow = refRow.iloc[0]
                except:
                    print(idenPage)
                    print(refRow)
            for col in refRow.index:
                if refRow[col]!=0:
                    self.scaledDict[idenPage][col] = (
                            self.scaledDict[idenPage][col]/refRow[col])
            self.fitDict[idenPage] = []
            for col in [i for i in self.scaledDict[idenPage].columns
                        if i in idenDict.keys() and i!=idenPage]:
                df = self.scaledDict[idenPage][[idenPage]].copy()
                df["myOnes"] = 1
                m, c = np.linalg.lstsq(df[[idenPage,"myOnes"]],
                                       self.scaledDict[idenPage][col])[0]
                Sq = ((self.scaledDict[idenPage][col]-1)**2).sum()
                self.fitDict[idenPage].append({"m":m, "c":c, "col":col,
                            "sq":Sq})
            #self.fitDict[idenPage].sort(key=(lambda x:np.abs(x["m"])))
            self.fitDict[idenPage].sort(key=(lambda x:np.abs(x["m"])),
                        reverse=True)
        self.displayOrder = [{"k":k, "o":np.abs(v[0]["m"])} for k,v
                     in self.fitDict.items()]
        self.displayOrder.sort(key=(lambda x:x["o"]),reverse=True)
        self.displayOrder = [i["k"] for i in self.displayOrder]
        self.displayOrder = [self.displayOrder[4*i:4*(i+1)] for i
                             in range(len(self.displayOrder))]
        self.displayOrder = [i for i in self.displayOrder if len(i)>0]

    def plotProfiles(self,showRows,showLimit=5, save = None,
                     style = None):
        if style is None:
            myStyle = "darkgrid"
        else:
            myStyle = style
        with sns.axes_style(style):
            fig, axs = plt.subplots(len(showRows), showLimit,
                                    sharex=True, figsize=(12,10))
            for row, i in zip(showRows,range(len(showRows))):
                for col, j in zip([i["col"] for i in self.fitDict[row]],
                                  range(showLimit)):
                    axs[i][j].scatter(self.scaledDict[row][row],
                       self.scaledDict[row][col])
                    axs[i][j].set_title(col)
                axs[i][0].set_ylabel(row, size='large')
            fig.tight_layout()
            if save is not None:
                fig.savefig(save)

    def plotRSS(self,showVars,showLimit=5, save = None, style = None):
        if style is None:
            myStyle = "darkgrid"
        else:
            myStyle = style
        with sns.axes_style(style):
            fig, axs = plt.subplots((len(showVars)-1)//showLimit+1,
                                    showLimit, sharex=True,
                                    figsize=(12,10))
            axs = trim_axs(axs, len(showVars))
            for myVar, i, ax in zip(showVars,range(len(showVars)),axs):
                temp = self.aggDF[self.aggDF["variable"]==myVar]
                temp = temp[[myVar,"RSS","LEVEL"]]
                avalableLevels = temp["LEVEL"].unique()
                for myLevel in avalableLevels:
                    ax.plot(temp[temp["LEVEL"]==myLevel][myVar],
                            temp[temp["LEVEL"]==myLevel]["RSS"])
                ax.set_yscale('log')
                ax.set_xscale('log')
                ax.set_title(myVar)
            fig.tight_layout()
            if save is not None:
                fig.savefig(save)

class timeCourseVisualiser:
    def __init__(self,data):
        if not isinstance(data, list):
            data = [data]
        runningList=[]
        for i in range(len(data)):
            valueColumns=list(data[i].columns)
            if "Time" in valueColumns:
                valueColumns.remove("Time")
            if "" in valueColumns:
                valueColumns.remove("")
            dataFrame=pd.melt(data[i],id_vars=["Time"],
                    value_vars=valueColumns)
            dataFrame['index']=i
            runningList.append(dataFrame)
        self.longData = pd.concat(runningList,ignore_index=True)

    def multiPlot2(self,indexSelect=None,varSelect=None,wrapNumber=5,
                  compLines=None, save = None):
        df = self.longData
        if varSelect is not None:
            if not isinstance(varSelect, list):
                varSelect = [varSelect]
            df = df[df['variable'].isin(varSelect)]
        if indexSelect is not None:
            if not isinstance(indexSelect, list):
                indexSelect = [indexSelect]
            df=df.loc[df['index'].isin(indexSelect)]
        indexes = list(df['index'].unique())
        colors = sns.husl_palette(len(indexes)).as_hex()
        if compLines is not None:
            compVars=list(compLines.columns)
            dfB = compLines.copy()
            dfB["Time"]=dfB.index
            dfB = pd.melt(dfB,id_vars=["Time"],
                          value_vars=compVars)
            dfB["index"]=-1
            df = pd.concat([dfB,df],ignore_index=True)
            colors.append("#000000")
            indexes.append(-1)
        #colors = dict(zip(indexes, colors))
        grid = sns.FacetGrid(df, col="variable", col_wrap=wrapNumber,
                             palette=colors)
        grid.map(sns.lineplot,"Time","value","index")
        if save is not None:
            grid.savefig(save)

    def multiPlot(self,indexSelect=None,varSelect=None,wrapNumber=5,
                  compLines=None, save = None, xlim = None,
                  forceYAxisZero = True, colourOverride = None,
                  style = None, legend = None, varAsAxis = False,
                  xAxisLabel = None, yAxisLabel = None, figsize = (12,10),
                  legendLoc = 'lower right'):
        """Plots grid if time course variables

        Creats grid of rainbow coloured plots for each variable in
        colection of TimeCourses. Optionaly plots refrence data against
        it as black dots.

        Kwargs:
           indexSelect (list of ints): Indexs of time courses that should
               be included in the plot.
           varSelect (list of str): Variables from the Time course that
               should be graphed.
           wrapNumber (int): max number of columns to alow in grid.
           compLines (dataFrame): refrence data to graph against time
               courses. Sould be in wide format and have the times as the
               index.
           save (str): A path to where to save the image. if omited image
               not saved
        """
        if isinstance(compLines,list):
            compVars = [list(i.columns) for i in compLines]
            dfB = [i.copy() for i in compLines]
            for i in range(len(compLines)):
                if "Time" not in compVars[i]:
                    dfB[i]["Time"]=dfB[i].index
                else:
                    compVars[i].remove("Time")
                dfB[i] = pd.melt(dfB[i], id_vars=["Time"],
                   value_vars=compVars[i])
        elif compLines is not None:
            compVars=list(compLines.columns)
            dfB = compLines.copy()
            if "Time" not in compVars:
                dfB["Time"]=dfB.index
            else:
                compVars.remove("Time")
            dfB = pd.melt(dfB,id_vars=["Time"],
                          value_vars=compVars)
        if varSelect is None:
            varSelect=list(self.longData['variable'].unique())
        if indexSelect is None:
            indexSelect=list(self.longData['index'].unique())
        if not isinstance(indexSelect,list):
            indexSelect = [indexSelect]
        if len(varSelect)<wrapNumber:
            #cols = math.floor(math.sqrt(len(varSelect)))
            cols = math.ceil(math.sqrt(len(varSelect)))
        else:
            cols = wrapNumber
        rows = math.ceil(len(varSelect)/cols)
        if style is None:
            myStyle = "darkgrid"
        else:
            myStyle = style
        with sns.axes_style(style):
            fig, axs = plt.subplots(rows, cols, sharex=True,
                                        figsize=figsize)
            if (rows>1):
                axs = trim_axs(axs, len(varSelect))
            elif (cols==1):
                axs = [axs]
            if colourOverride is not None:
                myColorMap = plt.get_cmap(name="cool")
            else:
                myColorMap = plt.get_cmap(name="hsv",
                                          lut=len(indexSelect)+1)
            for ax, theVar, j in zip(axs, varSelect, range(len(varSelect))):
                if varAsAxis:
                    if isinstance(yAxisLabel,list):
                        ax.set_ylabel(theVar+" "+yAxisLabel[j])
                    elif yAxisLabel is not None:
                        ax.set_ylabel(theVar+" "+yAxisLabel)
                    else:
                        ax.set_ylabel(theVar)
                else:
                    ax.set_title(theVar)
                    if isinstance(yAxisLabel,list):
                        ax.set_ylabel(yAxisLabel[j])
                    elif yAxisLabel is not None:
                        ax.set_ylabel(yAxisLabel)
                if xAxisLabel is not None:
                    ax.set_xlabel(xAxisLabel)
                df = self.longData
                df = df[df['variable']==theVar]
                if indexSelect is not None:
                    for theIndex, i in zip(indexSelect,
                                           range(len(indexSelect))):
                        df2 = df[df['index']==theIndex]
                        if colourOverride is not None:
                            ax.plot(df2["Time"], df2["value"],
                                    linestyle='solid',
                                    color=myColorMap(colourOverride[i]))
                        else:
                            ax.plot(df2["Time"], df2["value"],
                                    linestyle='solid',
                                    color=myColorMap(i))
                if isinstance(compLines,list):
                    for i, theIndex in enumerate(indexSelect):
                        dfB2 = dfB[theIndex][
                                dfB[theIndex]['variable']==theVar]
                        if colourOverride is not None:
                            ax.plot(dfB2["Time"], dfB2["value"],"o",
                                    color=myColorMap(colourOverride[i]))
                        else:
                            ax.plot(dfB2["Time"], dfB2["value"],"o",
                                    color=myColorMap(i))
                elif compLines is not None:
                    dfB2 = dfB[dfB['variable']==theVar]
                    ax.plot(dfB2["Time"], dfB2["value"],"ko")
                if xlim is not None:
                    ax.set_xlim(xlim)
                if forceYAxisZero:
                    ax.set_ylim([0, None])
            if legend is not None:
                if colourOverride is not None:
                    custom_lines = [Line2D([0], [0], color=myColorMap(
                            colourOverride[i]), lw=4)
                            for i in range(len(indexSelect))]
                else:
                    custom_lines = [Line2D([0], [0], color=myColorMap(i),
                                           lw=4)
                            for i in range(len(indexSelect))]
                if ((not isinstance(compLines,list)) and
                    (compLines is not None)):
                    custom_lines.append(Line2D([0], [0],
                                               color="k", lw=4))
                fig.legend(custom_lines, legend,
                           loc = legendLoc)
            fig.tight_layout()
            if save is not None:
                fig.savefig(save)

    def barChart(self, time, indexSelect=None, varSelect=None,
                 wrapNumber=5, compLines=None, save = None,
                 style = None, colourOveride = None, varOnAxis = False,
                 spacing = None, figsize=(12,10)):
        if compLines is not None:
            compVars=list(compLines.columns)
            dfB = compLines.copy()
            if "Time" not in compVars:
                dfB["Time"]=dfB.index
            else:
                compVars.remove("Time")
            dfB = pd.melt(dfB,id_vars=["Time"],
                          value_vars=compVars)
        if varSelect is None:
            varSelect=list(self.longData['variable'].unique())
        if indexSelect is None:
            indexSelect=list(self.longData['index'].unique())
        if isinstance(indexSelect,dict):
            renameIndex = [v for _,v in indexSelect.items()]
            indexSelect = [k for k,_ in indexSelect.items()]
        elif not isinstance(indexSelect,list):
            indexSelect = [indexSelect]
            renameIndex = None
        if len(varSelect)<wrapNumber:
            cols = math.floor(math.sqrt(len(varSelect)))
        else:
            cols = wrapNumber
        rows = math.ceil(len(varSelect)/cols)
        if style is None:
            myStyle = "darkgrid"
        else:
            myStyle = style
        with sns.axes_style(myStyle):
            fig, axs = plt.subplots(rows, cols, sharex=True,
                                    figsize=figsize)
            if (rows>1):
                axs = trim_axs(axs, len(varSelect))
            elif (cols==1):
                axs = [axs]
            # may need to add black
            myColorMap = plt.get_cmap(name="hsv", lut=len(indexSelect)+1)

            for ax, theVar in zip(axs, varSelect):
                df = self.longData
                df = df[df['variable']==theVar]
                df = df[df['Time']==time]
                df = df[df['index'].isin(indexSelect)]
                if spacing is not None:
                    bar_pos = np.arange(len(df['index'])+sum(spacing))
                    bar_pos = [i for i,j in zip(bar_pos,spacing) if not j]
                else:
                    bar_pos = np.arange(len(df['index']))
                if colourOveride is None:
                    colorList = [[j for j,x in enumerate(indexSelect)
                                  if x == i][0] for i in df['index']]
                    colorList = [myColorMap(i) for i in colorList]
                else:
                    colorList = colourOveride
                ax.bar(bar_pos, df["value"], color=colorList)
                if varOnAxis:
                    ax.set_ylabel(theVar)
                else:
                    ax.set_title(theVar)
                ax.set_xticks(bar_pos)
                ax.set_xticklabels(renameIndex)
                if compLines is not None:
                    df = dfB[dfB["Time"] == time]
                    df = df[df['variable'] == theVar]
                    if len(df)==1:
                        ax.axhline(y=df.squeeze()["value"])
            fig.tight_layout()
            if save is not None:
                fig.savefig(save)

class parameterEstimationVisualiser:
    def __init__(self,data):
        if not isinstance(data, list):
            data = [data]
        BPList=[]
        RSSList=[]
        wideList=[]
        for i in range(len(data)):
            df = data[i][list(data[i])[0]].copy()
            theColumns=list(df.columns)
            if "" in theColumns:
                df=df.drop(columns="")
            theColumns=list(df.columns).remove('RSS')
            df['subIndex'] = df.index
            df=pd.melt(df,id_vars=['subIndex','RSS'],
                       value_vars=theColumns)
            df['index']=i
            BPList.append(df)
            df = data[i][list(data[i])[0]].copy()
            df['subIndex'] = df.index
            df=df[['subIndex',"RSS"]]
            df['index']=i
            RSSList.append(df)
            df = data[i][list(data[i])[0]].copy()
            df['index']=i
            wideList.append(df)
        self.BPData = pd.concat(BPList,ignore_index=True)
        self.RSSData = pd.concat(RSSList,ignore_index=True)
        self.wideData = pd.concat(wideList,ignore_index=True)

    def violinPlot(self,indexSelect=None,paramSelect=None,RSSSelect=None,
                   save = None, indexNames=None):
        """Creates Violin plot showing variation in paramiter space

        Kwargs:
           indexSelect (list of ints): Indexs of sets of paramiter
               estimation results that should be ploted. Takes a maximum of
               2 to allow comparision of two sets. optionaly a single
               interger may be used. Omision leads to all sets of paramiter
               estimations being processed in a combined way.
           varSelect (list of str): Variables from the paramiter
               estimations that should be graphed.
           RSSSelect (list of floats): Specifies a maximum RSS value for
               data in the paramiter estimations to be included in the plot.
               Ordering should match indexSelect.
           save (str): A path to where to save the image. if omited image
               not saved
           indexNames (list of str): names to use in the plot legend. If
               Omited numerical index is used. Ordering should match
               indexSelect.
        """
        df=self.BPData
        if RSSSelect is not None:
            if not isinstance(RSSSelect, list):
                RSSSelect = [RSSSelect]
            if indexSelect is None:
                df = df.loc[df['RSS']<=RSSSelect[0]]
        if indexSelect is not None:
            if not isinstance(indexSelect, list):
                indexSelect = [indexSelect]
            if RSSSelect is not None:
                for i in range(len(indexSelect)):
                    df = df.loc[(df['RSS']<=RSSSelect[i]) |
                            (df['index']!=indexSelect[i])]
            df = df.loc[df['index'].isin(indexSelect)]
        if paramSelect is not None:
            if not isinstance(paramSelect, list):
                paramSelect = [paramSelect]
            df = df.loc[df['variable'].isin(paramSelect)]
        plt.figure()
        if indexNames is not None:
            if indexSelect is not None:
                if len(indexSelect)!=len(indexNames):
                    return None
            df = df.copy()
            df["index"] = df["index"].replace(
                    dict(zip(indexSelect,indexNames)))
        if indexSelect is None:
            if paramSelect is None:
                vp = sns.violinplot(x="variable", y="value", data=df, cut=0)
            else:
                vp = sns.violinplot(x="variable", y="value", data=df,
                                    order=paramSelect, cut=0)
        else:
            if paramSelect is None:
                vp = sns.violinplot(x="variable", y="value", data=df,
                                    hue="index", split=True, cut=0)
            else:
                vp = sns.violinplot(x="variable", y="value", data=df,
                                    hue="index", split=True,
                                    order=paramSelect, cut=0)
        if save is not None:
            vp.get_figure().savefig(save)

    def waterFall(self,save = None, indexNames=None, style = None):
        """creates watrfall plot of RSS values in paramiter estimations

        Kwargs:
           save (str): A path to where to save the image. if omited image
               not saved
           indexNames (list of str): names to use in the plot legend.
        """
        if indexNames is not None:
            if not isinstance(indexNames,list):
                indexNames = [indexNames]
            df = self.RSSData.copy()
            df["index"] = [indexNames[x] for x in df["index"]]
        else:
            df = self.RSSData
        if style is None:
            myStyle = "darkgrid"
        else:
            myStyle = style
        with sns.axes_style(style):
            plt.figure()
            lp = sns.lineplot(data=df,x="subIndex",y="RSS",hue="index")
            if save is not None:
                lp.get_figure().savefig(save)

    def refPointVsRSS(self, myRef, save=None, indexNames=None):
        names=list(myRef.index)
        refPoint = myRef.copy()
        refPoint["index"] = 1
        refPoint["RSS"] = 1
        diffTable = self.wideData.copy()
        diffTable = (diffTable/refPoint)-1
        diffTable["index"] = diffTable["index"]+1
        diffTable["index"] = diffTable["index"].astype(int)
        diffTable["RSS"] = diffTable["RSS"]+1
        diffTable["difrence"] = 0
        for name in names:
            diffTable["difrence"] = (diffTable["difrence"] +
                     diffTable[name]*diffTable[name])
        diffTable["difrence"] = diffTable["difrence"] / len(names)
        if indexNames is not None:
            if not isinstance(indexNames,list):
                indexNames = [indexNames]
            diffTable["index"] = [indexNames[x] for x in diffTable["index"]]
        plt.figure()
        sp = sns.scatterplot(data=diffTable, x="RSS", y="difrence",
                             hue="index")
        if save is not None:
            sp.get_figure().savefig(save)


from airflow.utils.decorators import apply_defaults

from presidio.utils.airflow.operators.retention.retention_operator import RetentionOperator

class AlertRetentionOperator(RetentionOperator):
    """
    Runs an alert retention task (a JAR file) using a bash command.
    The c'tor accepts the task arguments that are constant throughout the
    operator runs (e.g. the fixed duration strategy and the schema).
    Other arguments, such as the start date and the end date, are evaluated before every run.
    """

    @apply_defaults
    def __init__(self, command, entity_type, task_id=None, *args, **kwargs):
        """
        C'tor.
        :param command: command
        :type command: string
        :param entity_type: the entity type to process the alert cleanup.
        :type entity_type: string
        :param task_id: The task ID of this operator - If None, the ID is generated automatically
        :type task_id: string
        """
        self.log.debug('alert retention operator init kwargs=%s', str(kwargs))
        self.task_id = task_id or 'alert_retention'
        self.log.debug('agg operator. command=%s', command)
        java_args = {'entity_type': entity_type}

        super(AlertRetentionOperator, self).__init__(
            task_id=self.task_id,
            command=command,
            java_args=java_args,
            *args,
            **kwargs
        )


from django.conf.urls import url

from . import views

user_urls = [
    url(r'^(?P<pk>[0-9]+)/?$', views.UserDetail.as_view(), name='user-detail'),

    url(r'token', views.GetAuthToken.as_view(), name='api-token'),

    url(r'^$', views.UserList.as_view()),
]


import os
import requests
import uuid

from . import app, logger
from toutiao.crawler import Crawler
from model import Article
from video.tts import Txt2Mp3
from video.video import img2mp4
from ytb.utils import video_in_path, upload_toutiao
from service.auth import Auth

@app.task(bind=True, queue='queue_toutiao')
def fetch_upload(self, *args, **kwargs):
    article_id = Crawler().fetch_one()
    article = Article.get_by_id(article_id)
    logger.info('fetched article {}'.format(article.title))

    content = article.content
    title = article.title
    image_urls = article.image_urls

    asset_path = '/tmp/{}'.format(uuid.uuid4().hex[:6])
    os.mkdir(asset_path)
    file_path = os.path.join(asset_path, 'asset.txt')
    with open(file_path, 'w') as f:
        f.write(content)

    for i, url in enumerate(image_urls):
        response = requests.get(url)
        if response.status_code == 200:
            img_path = os.path.join(asset_path, '{}.jpg'.format(i))
            with open(img_path, 'wb') as f:
                f.write(response.content)
                logger.info('pic downloaded {}'.format(img_path))

    t2m = Txt2Mp3(asset_path)
    t2m.run()
    img2mp4(asset_path)

    title = title.split('。')[0]
    if '】' in title:
        title = title.split('】')[0] + '】'

    video = video_in_path(asset_path)
    credentials = Auth.get_credentials_by_account_id(2)
    client = Auth.get_client(credentials)
    upload_toutiao(client, video, title)


from  matplotlib import pyplot as plt
import numpy as np

model_fnames = {
    'crust': 'rho.crust{}.dat',
    'prem': 'rho.prem{}.dat',
}

import argparse
parser=argparse.ArgumentParser()
group = parser.add_mutually_exclusive_group(required=True)
group.add_argument('--xlength', nargs=2, metavar=('MIN', 'MAX'),type=float)
group.add_argument('--xdepth', nargs=2, metavar=('MIN', 'MAX'),type=float)
parser.add_argument("--yrho",nargs=2,metavar=('MIN','MAX'),type=float)
parser.add_argument("--curve",type=int,required=True)
parser.add_argument("--model", choices=model_fnames.keys())
parser.add_argument("--plot-curves", action='store_true')
parser.add_argument("--plot-crust", action='store_true')
parser.add_argument("--plot-avg", action='store_true')
parser.add_argument("--plot-std", action='store_true')
parser.add_argument("--plot-min", action='store_true')
parser.add_argument("--plot-max", action='store_true')

args=parser.parse_args()
ax=plt.gca()
plt.minorticks_on()
if args.yrho is not None:
    ax.set_ylim(*args.yrho)

if args.xlength:
    min_length, max_length = args.xlength
    min_rho,max_rho=args.yrho
    plt.xlabel('length  (km)')
    plt.ylabel(r'$\rho$  (g/cm$^{3}$)')
    plt.title('Density distribution in Earth matter')

fname=model_fnames[args.model]
lall = []
funcs = []
for i in range(0,args.curve+1):
    rho, ls = np.loadtxt(fname.format(i),float), np.loadtxt('length{}.dat'.format(i),float)
    ls = np.hstack([-ls[ls<=0][0:-1], 2*ls.max()-ls[ls>=0][1:]])
    lall.append(ls)

    funcs.append(lambda l, ls=ls, rho=rho: rho[np.searchsorted(ls, l, side='right')-1])

    kwargs = {}
    if i == 0:
        kwargs['linewidth'] = 2
    if args.plot_curves or i == 0 and args.plot_crust:
        plt.plot(ls, rho, label=('crust' if i == 0 else None))
    ax.set_xlim(min_length, max_length)

lall = np.unique(sorted(np.hstack(lall)))
rhoall = []
for func in funcs:
    rhoall.append(func(lall))
rhoall = np.vstack(rhoall).T
rhoavg = rhoall.mean(axis=1)
if args.plot_avg:

    ax.plot(lall, rhoavg, label='average')
if args.plot_min:
    ax.plot(lall, np.amin(rhoall, axis=1), label='min')
if args.plot_max:
    ax.plot(lall, np.amax(rhoall, axis=1), label='max')
if args.plot_std:
    rhostd = rhoall.std(axis=1)
    ax.fill_between(lall, rhoavg-rhostd, rhoavg+rhostd, alpha=0.3)

plt.legend(loc='lower center')
if args.xdepth:
    rho, d = np.loadtxt("rho.dat"), np.loadtxt("depth.dat")
    min_depth, max_depth = args.xdepth
    min_rho,max_rho=args.yrho
    pmask = np.logical_and(min_depth < d, d <= max_depth)
    prho, pl = rho[pmask], d[pmask]
    mmask =np.logical_and(-max_depth <= d, d < -min_depth)
    mrho, ml = rho[mmask], d[mmask]

    plt.xlabel('depth  (km)')
    plt.ylabel(r'$\rho$  (g/cm$^{3}$)')
    plt.title('Density distribution in Earth matter')
    ax.set_xlim(-(max_depth-min_depth), max_depth-min_depth)
    plt.plot(ml-ml.max(), mrho)
    plt.plot(pl-pl.min(), prho)
    ticks = plt.xticks()[0]
    ticks[ticks<0] = -ticks[ticks<0]
    ticks+=min_depth

ax.grid(which='both')
ax.grid(which='minor', alpha=0.5)
ax.grid(which='major', alpha=0.5)

plt.show()


from argparse import ArgumentParser
from tqdm import tqdm
from tensorboardX import SummaryWriter
import shutil
import os
import torch
import torch.nn as nn

from transfer.module.seq2seq import CycleLoss
from transfer.module.vae import (
    VariationalEncoder, VariationalDecoder, GaussianKLLoss)
from transfer.dataset import PttDataset, DcardDataset
from transfer.utils import Voc, transfer, straight_through_estimate
from transfer.constant import pattern, Constants

# cvae
max_length = 50
embedding_dim = 128
latent_dim = w    # content dimension
style_dim = 64      # style dimension
num_layers = 1
dropout = 0.1
bidirectional = True
st_mode = False
hidden_size = 256
# train
epochs = 1
batch_size = 32
chunk_size = 50
# loss coffecient
alpha = 1.          # reconstruct loss
gamma = 1.          # KL div
# gradient clipping
max_grad_norm = 1
# model path
model_dir = './models/%s'
log_dir = './log/%s'

D_threshold = 0.5

def save(E, G, model_name):
    path = os.path.join(model_dir, model_name)
    os.makedirs(path, exist_ok=True)
    torch.save(E.state_dict(), os.path.join(path, 'E.pt'))
    torch.save(G.state_dict(), os.path.join(path, 'G.pt'))

def train(batch_x, batch_y, E, G,
          criterion_cyc, criterion_KL, optimizer_G, num_iter, device):
    real_x, style_x = batch_x['seq'].to(device), batch_x['style'].to(device)
    real_y, style_y = batch_y['seq'].to(device), batch_y['style'].to(device)
    loss = {}

    # encode
    outputs_x, latent_x, mean_x, std_x = E(real_x, style_x, device)
    outputs_y, latent_y, mean_y, std_y = E(real_y, style_y, device)
    # decode
    rect_x = G(latent_x, style_x, outputs_x, max_length, device)
    rect_y = G(latent_y, style_y, outputs_y, max_length, device)
    fake_x = G(latent_y, style_x, outputs_y, max_length, device)
    fake_y = G(latent_x, style_y, outputs_x, max_length, device)
    fake_x = straight_through_estimate(fake_x)
    fake_y = straight_through_estimate(fake_y)

    # ===== G =====
    # reconstruct loss
    G_loss_cyc_x = criterion_cyc(rect_x, real_x)
    G_loss_cyc_y = criterion_cyc(rect_y, real_y)
    G_loss_cyc = (G_loss_cyc_x + G_loss_cyc_y) / 2
    # KL
    G_loss_KL_x = criterion_KL(mean_x, std_x)
    G_loss_KL_y = criterion_KL(mean_y, std_y)
    G_loss_KL = (G_loss_KL_x + G_loss_KL_y) / 2
    # KL loss
    G_loss = alpha * G_loss_cyc + gamma * G_loss_KL
    # optimize G
    optimizer_G.zero_grad()
    G_loss.backward()
    torch.nn.utils.clip_grad_norm_(E.parameters(), max_grad_norm)
    torch.nn.utils.clip_grad_norm_(G.parameters(), max_grad_norm)
    optimizer_G.step()
    # update loss
    loss.update({
        # G loss
        'G/loss_cyc_x': G_loss_cyc_x.item(),
        'G/loss_cyc_y': G_loss_cyc_y.item(),
        'G/loss_KL_x': G_loss_KL_x.item(),
        'G/loss_KL_y': G_loss_KL_y.item(),
        'G/loss_cyc': G_loss_cyc.item(),
        'G/loss_KL': G_loss_KL.item(),
        'G/loss': G_loss.item(),
    })

    return loss

def write_example(E, G, batch_x, batch_y, writer, num_iter, device):
    ptt2ptt = []
    mapping = Constants.idx2example()
    for seq_x in batch_x['seq'].numpy():
        real_x = Voc.decode(seq_x, mapping=mapping)
        rect_x = transfer(E, G, real_x, 'ptt', 'ptt', max_length, device)
        real_x = Voc.decode(seq_x)
        ptt2ptt.append(pattern % (real_x, rect_x))
    writer.add_text('PTT_to_PTT', '\n'.join(ptt2ptt), num_iter)

    dcard2dcard = []
    for seq_y in batch_y['seq'].numpy():
        real_y = Voc.decode(seq_y, mapping=mapping)
        rect_y = transfer(E, G, real_y, 'dcard', 'dcard', max_length, device)
        real_y = Voc.decode(seq_y)
        dcard2dcard.append(pattern % (real_y, rect_y))
    writer.add_text('DCARD_to_DCARD', '\n'.join(dcard2dcard), num_iter)

if __name__ == '__main__':
    parser = ArgumentParser('Train unsupervised style transfer model')
    parser.add_argument(
        '--load_pretrain', help="load pretrained autoencoder",
        action='store_true')
    parser.add_argument(
        '--name', help='the name of model', default="CVAE")
    args = parser.parse_args()
    model_dir = model_dir % args.name
    log_dir = log_dir % args.name
    shutil.rmtree(log_dir, ignore_errors=True)
    writer = SummaryWriter(log_dir=log_dir)

    # load dataset
    x_dataloader = torch.utils.data.DataLoader(
        dataset=PttDataset(chunk_size=chunk_size),
        batch_size=batch_size, shuffle=True, drop_last=True)
    y_dataloader = torch.utils.data.DataLoader(
        dataset=DcardDataset(chunk_size=chunk_size),
        batch_size=batch_size, shuffle=True, drop_last=True)
    voc = Voc()

    device = torch.device('cuda:0' if torch.cuda.is_available() else 'cpu')

    embedding = nn.Embedding(Voc.vocab_size, embedding_dim)
    E = VariationalEncoder(
        embedding, latent_dim, hidden_size, num_layers,
        dropout, bidirectional).to(device)
    G = VariationalDecoder(
        embedding, style_dim, latent_dim, hidden_size, num_layers,
        dropout, st_mode).to(device)

    criterion_cyc = CycleLoss()
    criterion_KL = GaussianKLLoss()

    optimizer_G = torch.optim.RMSprop(
        list(E.parameters()) + list(G.parameters()), lr=0.001)

    num_iter = 0
    loss = {'G/loss': -1}
    for epoch in range(epochs):
        batch_num = min(len(x_dataloader), len(y_dataloader))
        with tqdm(total=batch_num) as t:
            t.set_description('Epoch %2d/%2d' % (epoch + 1, epochs))
            for batch_x, batch_y in zip(x_dataloader, y_dataloader):
                loss_new = train(
                    batch_x, batch_y, E, G, criterion_cyc, criterion_KL,
                    optimizer_G, num_iter, device)
                loss.update(loss_new)

                num_iter += 1
                if num_iter % 1000 == 1:
                    write_example(
                        E, G, batch_x, batch_y, writer, num_iter, device)

                for loss_name, value in loss.items():
                    writer.add_scalar(loss_name, value, num_iter)
                t.update(1)
                t.set_postfix(G_loss='%.4f' % loss['G/loss'])
        save(E, G, 'epoch%d' % (epoch + 1))
    write_example(E, G, batch_x, batch_y, writer, num_iter, device)
    writer.close()
    save(E, G, 'final')


from django import http
from django.conf import settings
from django.template import RequestContext, loader
from django.contrib.sites.models import Site

def page_not_found(request, template_name='perfect404.html'):
    """
    Perfect 404 handler.

    Templates: `perfect404.html`
    Context:
        request_path
            The path of the requested URL (e.g., '/app/pages/bad_page/')
    """
    referer = request.META.get('HTTP_REFERER', '')

    internal = False
    iam = 'http://%s' % Site.objects.get_current().domain
    if referer[:len(iam)] == iam:
        internal = True

    if settings.ADMINS:
        contact = dict(zip(('name', 'email'), settings.ADMINS[0]))
    else:
        contact = None

    t = loader.get_template(template_name)
    return http.HttpResponseNotFound(t.render(RequestContext(
                    request, {
                        'request_path': request.path,
                        'referer': referer,
                        'internal': internal,
                        'contact': contact,
                    })))