#!/usr/bin/env python
# -*- coding: utf-8 -*-

"""
easymail

an abstraction layer on top of the standard email package.
It aims to hide all the troublesome mime stuff and sets some sane defaults.
All to make your life a little bit easier and make sending emails easier.

>>> e = Email('sender@foo.com', 'recipient@foo.com')
>>> e.subject = 'urgent!!!1'
>>> e.body = 'this is very important.'
>>> e.attachments.append(Attachment('/path/to/audiofile.ogg'))

>>> e.sender
'sender@foo.com'
>>> str(e.subject)
'urgent!!!1'
>>> e.recipients
['recipient@foo.com']
"""

import os
import logging
from mimetypes import guess_type
from email import encoders
from email.header import Header
from email.mime.audio import MIMEAudio
from email.mime.base import MIMEBase
from email.mime.image import MIMEImage
from email.mime.multipart import MIMEMultipart
from email.mime.text import MIMEText
from email.utils import formatdate

__all__ = ['Attachment', 'Email']
logger = logging.getLogger(__file__)

class Attachment(object):
    def __init__(self, path, mimetype=None, filename=None):
        """create an attachment

        :param path: path to the file on disk
        :param mimetype: (optional, will be guessed by fileextension
        :param filename: filename in the email,
                         if None the name from path will be used
        """

        #: full path to the attachment
        self.path = path

        #: mimetype of the attachment
        self.mimetype = mimetype or (guess_type(path)[0] or
                                     'application/octet-stream')
        #: filename of the attachment in the email
        self.filename = filename

    def as_msg(self):
        """return the mime msg.

        the results concrete type depends on the attachments mimetype.
        """
        maintype, subtype = self.mimetype.split('/')
        with open(self.path, 'rb') as fp:
            if maintype == 'image':
                msg = MIMEImage(fp.read(), _subtype=subtype)
            elif maintype == 'audio':
                msg = MIMEAudio(fp.read(), _subtype=subtype)
            else:
                msg = MIMEBase(maintype, subtype)
                msg.set_payload(fp.read())
                encoders.encode_base64(msg)

        msg.add_header('Content-Disposition',
                       'attachment',
                       filename=self.filename or os.path.basename(self.path))
        return msg

class Email(object):
    def __init__(self, sender, recipients, subject='', body=''):
        """

        :param sender: email of the sender. E.g. 'me@mydomain.com'
                       or in the format 'My Name <me@mydomain.com>'
        :param recipients: a :class:`list` of recipients.
                           ['one@recipient.com', 'Two <two@recipient.com>']
        :param subject: email subject
        :param body: email body.
                     If it contains HTML `body_is_html` needs to be set to True
        """
        self.sender = sender

        if isinstance(recipients, str):
            self.recipients = [recipients]
        else:
            self.recipients = recipients

        self.subject = subject
        self.attachments = []
        self.reply_to = ''
        self.cc = []
        self.bcc = []

        #: Mail body
        self.body = body

        self._body_is_html = False

    def __repr__(self):
        return '<Email>'

    def __str__(self):
        return self.get_msg()

    @property
    def body_is_html(self):
        """If set to `True` the email will be sent as HTML
        Otherwise a regular plain/text email will be sent.

        :returns: :class:`bool`
        """
        return self._body_is_html

    @body_is_html.setter
    def body_is_html(self, value):
        self._body_is_html = value

    @property
    def all_recipients(self):
        """recipients including cc and bcc"""
        return self.recipients + self.cc + self.bcc

    @property
    def subject(self):
        """subject of the email"""
        return str(self._subject)

    @subject.setter
    def subject(self, value):
        if isinstance(value, str):
            self._subject = Header(value, 'utf-8')
        else:
            self._subject = value
        assert isinstance(self._subject, Header)

    @property
    def args(self):
        """ *e.args can be used for :func:`smtplib.SMTP.sendemail`

        >>> e = Email('Sender <sender@mydomain.com>',
        ...           ['rec1@dom.com', 'rec2@dom.com'],
        ...           'subject',
        ...           'body')
        >>> e.args  # doctest:+ELLIPSIS
        ('Sender <sender@mydomain.com>', ['rec1@dom.com', 'rec2@dom.com'], ...

        :returns: :class:`tuple`
        """
        return (self.sender, self.all_recipients, self.get_msg())

    def get_msg(self):
        # python 3.3 doesn't like the utf-8 charset if the body is empty.
        charset = self.body != '' and 'utf-8' or 'us-ascii'

        if self.attachments or self.body_is_html:
            msg = MIMEMultipart()
        else:
            msg = MIMEText(self.body, _charset=charset)

        msg['Subject'] = self._subject
        msg['From'] = self.sender
        msg['To'] = ', '.join(self.recipients)
        if self.reply_to:
            msg['Reply-To'] = self.reply_to
        msg['Date'] = formatdate(localtime=True)

        if self.body_is_html:
            body = MIMEText(self.body, 'html', charset)
            msg.attach(body)

        if self.attachments:
            if not self.body_is_html:
                msg.attach(MIMEText(self.body, _charset=charset))

            for attachment in self.attachments:
                msg.attach(attachment.as_msg())

        result = str(msg)
        logger.debug(result)
        return result

if __name__ == "__main__":
    import doctest
    doctest.testmod()


import os
from requests import get
from io import BytesIO
from PIL import Image
from typing import Optional

class ImageDownloader:
    def __init__(self, link_dict: dict):
        self.links = link_dict

    def _switch_directory(self, folder_name: Optional[str] = None):
        """
        Checks the file path. If it is in the wrong directory,
        it comes back and enters the correct directory.
        If it is in the main directory(images), it enters the specified directory

        images: -> Main Folder
            icon_folder -> Subfolder
                icon.png
            badge_folder -> Subfolder
                badge.png
        """
        curr_path = os.getcwd().split('/')[-1]

        if (folder_name and folder_name == curr_path) or (not folder_name and curr_path == "images"):
            return
        elif folder_name and curr_path != "images":
            target_directory = '../' + folder_name
        elif not folder_name and curr_path != "images":
            target_directory = '..'
        else:
            target_directory = folder_name

        os.chdir(target_directory)

    def restore_directory(self):
        curr_path = os.getcwd().split('/')
        if curr_path[-1] == "images":
            old_directory = "../../../"
        else:
            old_directory = "../../../../"

        os.chdir(old_directory)

    async def start_download(self):
        """Download images in database to specified directories"""
        # Set directory
        os.chdir("cogs/assets")
        os.makedirs("images", mode=0o777, exist_ok=True)
        os.chdir("images")
        os.makedirs("expense_icons", mode=0o777, exist_ok=True)
        os.makedirs("badges", mode=0o777, exist_ok=True)

        try:
            for key, value in self.links.items():
                response = get(value)
                img = Image.open(BytesIO(response.content)).convert("RGBA")

                # Change directory
                if key[:4] == "icon":
                    self._switch_directory("expense_icons")
                elif key[:5] == "badge":
                    self._switch_directory("badges")
                else:
                    self._switch_directory()

                img.save(f'{key}.png')
            self.restore_directory()
            return True
        except Exception as err:
            print(err)
            return False


"""
pyaircraft.io.read_p3_flight
=========================

This is a grouping of scripts designed to process NOAA P-3
 flight level data recorded during flights and put into NetCDF
 format by NOAA AOC.

Created by Nick Guy.

"""
# NOTES:: This has only been tested with DYNAMO data files, versions
#         may change and another function may be needed.
# HISTORY::
#   8 Jan 2014 - Nick Guy.   NRC, NOAA/NSSL (nick.guy@noaa.gov)
#                Converted NCL functions below to Python
# FUNCTIONS::
#  flight_level_variable - Read in a variable from flight level NetCDF
#  flight_track - Read in data to for flight track
#-------------------------------------------------------------------
# Load the needed packages
from scipy.io import netcdf
from netCDF4 import Dataset,num2date
import json
import numpy as np
import pytz
#-------------------------------------------------------------------
# Define various constants that may be used for calculations
#===============================================================
# BEGIN FUNCTIONS
#===============================================================
def flight_level_variable(fname,Rec):
    """Read in data from NetCDF file containing P3 flight level data created
  by NOAA AOC.  The NetCDF should be read in the main program and passed
  to this function.
  A call such as this can be used in the main program:
      FltncID=addfile(FlightFileStringName,"r")
 INPUT::
  fname           = Filename [string]
  Rec             = Variable name to be pulled out [string]
 OUTPUT::
  VarOut          = Masked array containing variable data
 USAGE::
  Lat = read_flight_level_dynamo('P3.nc','LatGPS.3')
 NOTES::
 Data file structure::
  Available variables (not full list) :
  LonGPS.3      = Novatel GPS Longitude
  LatGPS.3      = Novatel GPS Latitude
  AltGPS.3      = Novatel GPS Altitude [m]
  THdgI-GPS.1   = True heading [deg]
  TRK.1         = Track [deg]
  AltPaADDU.1   = Pressure altitude [m]
  WSZ_DPJ       = Vertical wind via D Jorgensen calculation [m/s]
  TA.1          = Ambient Temperature [C]
  TD.1          = Dewpoint Temperature [C]
  TVIRT.1       = Virtual Temperature [K]
  THETA.1       = Potential Temperature [K]
  THETAE.1      = Equivalent Potential Temperature [K]
  THETAV.1      = Virtual Potential Temperature [K]
  WS.1          = Wind Speed [m/s]
  WD.1          = Wind Direction [deg]
  HUM_REL.1     = Relative Humidity [%]
  HUM_SPEC.1    = Specific Humidity [g/kg]
  MR.1          = Mixing ratio [g] [g/g?]
  EE.1          = Vapor Pressure [hPa]
  EW.1          = Saturated Vapor Pressure [hPa]
      """
# HISTORY::
#  25 Jul 2013 - Nick Guy NOAA/NSSL/WRDD, NRC
#   8 Jan 2014 - With change to python, the procedure of setting the bad
#                 data by user is no longer used, but the numpy masking
#                 technique is instead used to nuke "bad" occurences
#---------------------------------------------------
    # Read the NetCDF
    ncFile = netcdf.netcdf_file(fname,'r')

    # Get the variable of interest
    VarOut = ncFile.variables['Rec'][:]

    # Mask any "_FillValue" or some missing_data type attribute
    #try:
    #    VarOut = np.ma.masked_values(VarOut, VarOut.missing_value)
    #except:
    #    pass
    #try:
    #    VarOut = np.ma.masked_values(VarOut, VarOut.Missing_Value)
    #except:
    #    pass
    #try:
    #    VarOut = np.ma.masked_values(VarOut, VarOut._FillValue)
    #except:
    #    pass

    # Mask any NaN values
    #VarOut = np.ma.masked_values(VarOut, np.isnan(VarOut)

    return VarOut
#**====================================================
def flight_track(fname):
    """Read in data from NetCDF file containing P3 flight level data created
  by NOAA AOC.  Pull out the needed variables for flight track info.
 INPUT::
  fname           = Filename [string]
 OUTPUT::
  Lat             = Aircraft latitude
  Lon             = Aircraft longitude
  Alt             = Aircraft altitude
  PAlt            = Aircraft pressure altitude
  Time            = Aircraft time array
 USAGE::
  Lat,Lon,Alt,PAlt = flight_track(fname)
    """
# HISTORY::
#   7 Mar 2014 - Nick Guy NOAA/NSSL/WRDD, NRC
#---------------------------------------------------
    # Read the NetCDF
    ncFile = Dataset(fname,'r')

    # Pull out each variable
    Lat = ncFile.variables['LatGPS.3'][:]
    Lon = ncFile.variables['LonGPS.3'][:]
    Alt = ncFile.variables['AltGPS.3'][:]
    PAlt = ncFile.variables['AltPaADDU.1'][:]

    # Pull out the start time
    StartTime = ncFile.StartTime

    # Create a time array
    TimeSec = np.linspace(StartTime,StartTime + len(Lat), len(Lat))

    Time_unaware = num2date(TimeSec,'seconds since 1970-01-01 00:00:00+0:00')
    Time = Time_unaware#.replace(tzinfo=pytz.UTC)

    # Now mask missing values
    np.ma.masked_invalid(Lat)
    np.ma.masked_invalid(Lon)
    np.ma.masked_invalid(Alt)
    np.ma.masked_invalid(PAlt)

    return Lat,Lon,Alt,PAlt,Time


"""This module provides a simple tcp server to connect to the database."""
import argparse
import asyncio
import logging
import os
import signal
import socket
from functools import partial
import hnswlib

from cryptography.fernet import InvalidToken
from luxdb.connection import gen_key, receive_command, send_close, send_result
from luxdb.index import Index
from luxdb.knn_store import KNNStore, open_store

LOG = logging.getLogger('server')

def shutdown(server, signum):
    """Gracefully shut the server down."""
    LOG.info('%s, shutting down', signal.strsignal(signum))
    server.shutdown()

class Server:
    """TCP Server that allows clients to connect to the database.

    Receives commands, executes them on the database and sends the result back.

    """
    def __init__(self, host: str, port: int, store: KNNStore, secret):
        """Define the host and port where the server will listen, and the store that should be used."""
        self.host = host
        self.port = port
        self.store = store
        self.server = None
        self.secret = gen_key(secret)

    async def _next_command(self, reader, writer):
        try:
            return await receive_command(reader, self.secret)
        except InvalidToken:
            LOG.error('Invalid token received, closing connection.')
            await send_close(writer)
            writer.close()
            return None

    async def connection(self, reader: asyncio.StreamReader, writer: asyncio.StreamWriter):
        """Handles a single incoming connection asynchronously"""
        LOG.debug('New connection established.')

        LOG.debug('Waiting for command')
        command = await self._next_command(reader, writer)

        LOG.debug('Command received: %s', command)

        while command is not None:
            result = await command.execute(self.store)
            LOG.debug('Sending result to')

            await send_result(writer, result, self.secret)
            LOG.debug('Sent result to client.')

            LOG.debug('Waiting for command')
            command = await self._next_command(reader, writer)

        writer.close()
        await writer.wait_closed()

    async def create_server(self):
        """Creates the server socket but doesn't start listening yet."""
        self.server = await asyncio.start_server(self.connection, self.host, self.port)

        for server_socket in self.server.sockets:
            host, port = socket.getnameinfo(server_socket.getsockname(), socket.NI_NUMERICSERV | socket.NI_NUMERICHOST)
            LOG.info('Serving on %s:%s', host, port)
            self.host = host
            self.port = int(port)

    def shutdown(self):
        """Gracefully terminate the database server."""
        self.server.close()

    async def start(self, callback=None):
        """Start the server and process incoming connections.

        The callback is called shortly before the server starts listening, you can use it to notify
        clients that the server is listening, so they can connect without an error.
        """
        if self.server is None:
            await self.create_server()

        async with self.server:
            loop = asyncio.get_running_loop()
            if callback is not None:
                loop.call_soon(callback)

            await self.server.start_serving()
            await self.server.wait_closed()

async def serve(args: dict):
    """Main method for the server thread."""

    with open_store(args.path) as store:
        loop = asyncio.get_running_loop()
        server = Server(host=args.host, port=args.port, store=store, secret=args.secret)

        for signum in [signal.SIGTERM, signal.SIGINT]:
            loop.add_signal_handler(signum, partial(shutdown, server, signum))

        await server.start()

async def import_index(args: dict):
    """Command to import the index."""
    with open_store(args.path) as store:
        index = hnswlib.Index(space=args.import_space, dim=args.import_dim)
        index.load_index(args.import_path)
        index = Index(index)
        store.import_index(args.import_name, index)

def main():
    """Main method for the server."""
    parser = argparse.ArgumentParser(description='Multidimensional vector database (server).')

    parser.add_argument('--host', type=str, default='127.0.0.1', help='Host where the server should listen.')
    parser.add_argument('--port', type=int, default=None, help='Port to listen on.')
    parser.add_argument(
        '-log',
        '--loglevel',
        default='warning',
        help='Provide logging level. Example --loglevel debug, default=warning',
    )
    parser.add_argument(
        '--import-path',
        type=str,
        default=None,
        help="Don't run the server, but import an index. Specify the path",
    )
    parser.add_argument(
        '--import-name',
        type=str,
        default=None,
        help="Don't run the server, but import an index. Specify the name",
    )
    parser.add_argument(
        '--import-space',
        type=str,
        default=None,
        help="Don't run the server, but import an index. Specify the space",
    )
    parser.add_argument(
        '--import-dim',
        type=int,
        default=None,
        help="Don't run the server, but import an index. Specify the dimension",
    )
    parser.add_argument('--secret',
                        type=str,
                        default=os.environ.get('LUXDB_SECRET', ''),
                        help='Pass the secret the server should use, if not specified it is taken from $LUXDB_SECRET')
    parser.add_argument('path', type=str, help='Path where the database is stored or should be stored.')
    args = parser.parse_args()
    logging.basicConfig(level=args.loglevel.upper())

    if args.import_path is not None:
        return asyncio.run(import_index(args), debug=args.loglevel.upper() == 'DEBUG')

    return asyncio.run(serve(args), debug=args.loglevel.upper() == 'DEBUG')


#!/usr/bin/env python
# -*- coding: utf-8 -*-

from __future__ import print_function, unicode_literals, absolute_import

import connection as co
import readline, string, os, sys
import argparse

from pydoc import pager

try:
    input = raw_input
except:
    pass

FNAME = os.path.expanduser('~/.revterm_history')

WARNING = '\033[95m'
OKBLUE = '\033[94m'
OKGREEN = '\033[92m'
FAIL = '\033[91m'
ENDC = '\033[0m'

codes = {
    0: OKGREEN,
    1: OKGREEN,
    2: OKGREEN,
    3: OKGREEN,
    4: WARNING
}

class Prompt(object):
    def __init__(self):
        self.database = '<none>'
        self.count = 0
        self.format = string.Template('$database [$count] $prompt_char ')
        self.prompt_char = '>'

    def increment(self):
        self.count += 1

    def in_copy(self):
        self.prompt_char = '|'

    def out_copy(self):
        self.prompt_char = '>'

    def __str__(self):
        return OKBLUE + self.format.safe_substitute(self.__dict__) + ENDC

def present_prompt(p):
    try:
        value = input(str(p))
    except EOFError:
        result = 'k'

        while len(result) >= 1 and result[0] != 'y' and result[0] != 'n':
            try:
                result = input("\nDo you really want to exit? ([y]/n) ")
            except EOFError:
                result = 'y'

        if len(result) == 0 or result[0] == 'y':
            sys.exit(0)
        else:
            return present_prompt(p)
    except KeyboardInterrupt:
        print('')
        return present_prompt(p)
    return value

def generate_output(value):
    output = ''

    if isinstance(value.message, str):
        output += value.message
    else:
        for item in value.message:
            output += '\t' + '\t'.join(item) + '\n'

    return output

def send(con, msg):
    try:
        con.send(msg)
        return con.recv()
    except Exception as e:
        print('Client encountered fatal error. Exiting...')
        print(e)
        sys.exit(1)

def send_direct(con, msg):
    try:
        con.send(msg)
    except Exception as e:
        print('Client encountered fatal error. Exiting...')
        print(e)
        sys.exit(1)

def build_parser():
    parser = argparse.ArgumentParser(description='CLI Interface for Revori')

    parser.add_argument('-hs', '--host',  type=str, default='localhost')
    parser.add_argument('-p', '--port', type=int, default=8017)
    parser.add_argument('-n', '--lines', type=int, default=25,
        help='Maximum number of lines to display before paging.')

    return parser

def parse_args(args):
    parser = build_parser()

    return parser.parse_args(args)

def config_readline():
    with open(FNAME, 'a') as f:
        os.utime(FNAME, None)

    readline.set_history_length(500)
    readline.read_history_file(FNAME)

def printout(max_lines, result):
    output = generate_output(result)
    if max_lines > 0 and output.count('\n') < max_lines:
        output = codes.get(result.code, ENDC) + output + ENDC
        print(output)
    else:
        pager(output)

def main():
    args = parse_args(sys.argv[1:])

    config_readline()

    p = Prompt()
    c = co.RevoriConnection(args.host, args.port)

    copying = False

    print("Welcome to the Revori SQL client interface." + ENDC)
    print('Type "help" to get started.\n' + ENDC);
    try:
        while True:
            value = present_prompt(p)

            if value == 'exit' or value == 'quit':
                sys.exit(0)

            value = value.strip()

            if len(value) > 0:
                if not copying:
                    p.increment()

                    result = send(c, value)

                    printout(args.lines, result)

                    if hasattr(result, 'database'):
                        p.database = result.database

                    copying = result.copy_context

                    if copying:
                        p.in_copy()
                else:
                    send_direct(c, value)

                    if value == '\\.':
                        copying = False
                        p.out_copy()

                        result = c.recv()
                        output = generate_output(result)
                        printout(args.lines, result)

    finally:
        c.close()
        print(ENDC)

        readline.write_history_file(FNAME)

if __name__ == '__main__':
    main()


#!/usr/bin/env python
# -*- coding: utf-8 -*-
'''sunsitive word views'''

__author__ = "wenxiaoning(371032668@qq.com)"
__copyright__ = "Copyright of hopapapa (2017)."

from src.common import http_util
from src.config import BaseConfig
from src.api.sensitive_word.models import SensitiveWord
from src.common.http_util import return_model
from src.common.web.flask_snippets import crossdomain
from src.common.web.flask_snippets import jsonp

from flask import request
from flask import Blueprint

admin_sensitive_word_bp = Blueprint('admin_sensitive_word', __name__)

@admin_sensitive_word_bp.route('/sensitive_word/list.json', methods=['GET'])
@jsonp
def list():
    '''文章列表'''

    args = request.args

    page = http_util.get_param_int(args, 'page', BaseConfig.DEFAULT_PAGE)
    per_page = http_util.get_param_int(args, 'per_page',
                                       BaseConfig.DEFAULT_PER_PAGE)

    paginate = SensitiveWord.query_paginate(
        page=page,
        per_page=per_page
    )

    items = []
    for item in paginate.items:
        detail = item.to_json()
        # detail['id'] = "{}".format(detail['id'])
        items.append(detail)

    res = http_util.make_page_response(items, paginate.total, page,
                                       per_page)

    return return_model(
        data=res
    )

@admin_sensitive_word_bp.route('/sensitive_word/edit.json', methods=['POST'])
@crossdomain(origin='*', methods=['POST'], headers='Origin, Content-Type')
def edit():
    '''创建文章'''
    args = request.form
    id = args.get('id', None)

    if id:
        SensitiveWord.update_by_id(**args)
    else:
        SensitiveWord.create(**args)

    return return_model()

@admin_sensitive_word_bp.route('/sensitive_word/delete.json', methods=['POST'])
@crossdomain(origin='*', methods=['POST'], headers='Origin, Content-Type')
def delete():
    '''删除敏感词'''
    args = request.form
    id = args.get('id', None)

    SensitiveWord.update_by_id(
        id=id,
        is_del=1
    )

    return return_model()


# -*- coding: utf-8 -*-

from uuid import uuid1

from flask import Flask, jsonify
from flask_cors import CORS
from pre_request import pre, Rule

from code import ResponseCode, ResponseMessage
from log import logger
from utils import base64_to_img, create_date_dir

# 创建一个服务
app = Flask(__name__)
CORS(app, supports_credentials=True)

"""
# 方法功能说明
"""
@app.route(rule='/api/moduleName/methodName', methods=['POST'])
def methodName():

    # 参数校验
    rule = {
        "img": Rule(type=str, required=True),
        "file_name": Rule(type=str, required=False)
    }
    try:
        params = pre.parse(rule=rule)
    except Exception as e:
        logger.error(e)
        fail_response = dict(code=ResponseCode.RARAM_FAIL, msg=ResponseMessage.RARAM_FAIL, data=None)
        logger.error(fail_response)
        return jsonify(fail_response)

    # 获取参数
    image_b64 = params.get("img")
    file_name = params.get("file_name")
    if file_name is None:
        file_name = '{}.jpg'.format(uuid1())

    # 将base64字符串解析成图片保存
    img_root_path = "./img"
    img_base_path = create_date_dir(img_root_path)
    img_path = img_base_path + file_name
    try:
        base64_to_img(image_b64, img_path)
    except Exception as e:
        logger.error(e)
        fail_response = dict(code=ResponseCode.RARAM_FAIL, msg=ResponseMessage.RARAM_FAIL, data=None)
        logger.error(fail_response)
        return jsonify(fail_response)

    # 下面对保存的图片进行若干处理
    result = image_b64
    logger.info("测试日志记录")

    # 处理完成后删除生成的图片文件
    # os.remove(img_path)

    # 成功的结果返回
    success_response = dict(code=ResponseCode.SUCCESS, msg=ResponseMessage.SUCCESS, data=result)
    logger.info(success_response)
    return jsonify(success_response)

if __name__ == '__main__':
    # 解决中文乱码问题
    app.config['JSON_AS_ASCII'] = False
    # 启动服务 指定主机和端口
    app.run(host='0.0.0.0', port=5000, debug=False, threaded=True)


# -------------------------------------------------------------------------------------------------------------------
# Reducing the Dimensionality of the Data
# Defeating the curse of dimensionality: for a given sample size, there exists an optimal number of features for best
# classification performance

# -------------------------------------------------------------------------------------------------------------------
# Implementing Principal Component Analysis (PCA) in OpenCV

# PCA rotates all data points they align with the two axes that explains the spread of the data. PCA aims to transform
# the data to a new coordinate system by means of an orthogonal linear transformation.
# Project the data onto the new coordinate system such that the first coordinate has the greatest variance.
# This is called the first principal component

# Here is some random data drawn from a multivariate Gaussian:
import numpy as np
from sklearn import decomposition

mean = [20, 20]
cov = [[5, 0], [25, 25]]
np.random.seed(42)
x, y = np.random.multivariate_normal(mean, cov, 1000).T

# Plot this data using Matplotlib. Look at the spread:
import matplotlib.pyplot as plt
# plt.style.use('ggplot')
plt.figure(figsize=(10, 6))
plt.plot(x, y, 'o', zorder=1)
plt.axis([0, 40, 0, 40])
plt.xlabel('feature 1')
plt.ylabel('feature 2')
# plt.show()

# Format the data by stacking the `x` and `y` coordinates as a feature matrix `X`:
X = np.vstack((x, y)).T

# Create an empty array `np.array([])` as a mask, telling OpenCV to use all data points in the feature matrix.
# Compute PCA on the feature matrix `X`
import cv2
mu, eig = cv2.PCACompute(X, np.array([]))
# print(eig)

# Note the following looks complicated but is simply for demonstration purposes (showing directional arrows of PC.
# Plot the eigenvectors of the decomposition on top of the data:
plt.figure(figsize=(10, 6))
plt.plot(x, y, 'o', zorder=1)
plt.quiver(mean[0], mean[1], eig[:, 0], eig[:, 1], zorder=3, scale=0.2, units='xy')
plt.text(mean[0] + 5 * eig[0, 0], mean[1] + 5 * eig[0, 1], 'u1', zorder=5,
         fontsize=16, bbox=dict(facecolor='white', alpha=0.6))
plt.text(mean[0] + 7 * eig[1, 0], mean[1] + 4 * eig[1, 1], 'u2', zorder=5,
         fontsize=16, bbox=dict(facecolor='white', alpha=0.6))
plt.axis([0, 40, 0, 40])
plt.xlabel('feature 1')
plt.ylabel('feature 2')
# plt.show()

# PCA rotates the data so that the two axes (`x` and `y`) are aligned with the first two principal components.

# In OpenCV the PCAProject function both initiates and fits the transformed matrix to X2:
# X2 = cv2.PCAProject(X, mu, eig)

# Scikit: first instantiate using the `decomposition` module:
pca = decomposition.PCA(n_components=2)

# Now use the `fit_transform` method:
X2 = pca.fit_transform(X)

# The blob of data is rotated so that the most spread is along the `x` axis:
# Note the more even spread
plt.figure(figsize=(10, 6))
plt.plot(X2[:, 0], X2[:, 1], 'o')
plt.xlabel('first principal component')
plt.ylabel('second principal component')
plt.axis([-20, 20, -10, 10])
# plt.show()

# Implementing Independent Component Analysis (ICA) performs the same mathematical
# steps as PCA, but it chooses the components of the decomposition to be as independent as possible from each other.

# Again, first instantiate, then use the `fit_transform` method:
ica = decomposition.FastICA()
X2 = ica.fit_transform(X)

# Plot the projected data on the first two independent components:
plt.figure(figsize=(10, 6))
plt.plot(X2[:, 0], X2[:, 1], 'o')
plt.xlabel('first independent component')
plt.ylabel('second independent component')
plt.axis([-0.2, 0.2, -0.2, 0.2])
plt.show()


from builtins import range
from builtins import object
from proteus import Comm, Profiling
import numpy as np
import numpy.testing as npt
import pytest
import os

comm = Comm.init()
Profiling.procID = comm.rank()

class MockMat(object):
    """ petsc4py-based mock SuperLU Matrix for testing.  Filled like this:

    [ 1 1     ]
    [   1 2   ]
    [     1 3 ]
    [       1 ]
    """

    def __init__(self, n):
        from petsc4py import PETSc
        A = PETSc.Mat().create()
        A.setSizes(n)
        A.setType('aij')
        A.setUp()
        A.assemblyBegin()
        for i in range(n):
            A[i, i] = 1
            if i + 1 < n:
                A[i, i + 1] = i
        A.assemblyEnd()
        self.A = A

    def getCSRrepresentation(self):
        return self.A.getValuesCSR()

    def getSubMatCSRrepresentation(self, start, end):
        from petsc4py import PETSc
        ids = list(range(start, end))
        isg = PETSc.IS().createGeneral(ids)
        B = self.A.createSubMatrix(isg)
        return B.getValuesCSR()

@pytest.mark.LinearAlgebraTools
def test_vec_create():
    """test_vec_create

    Verifies that the proteus.LinearAlgebraTools.Vec constructor
    correctly creates one-dimensional arrays of the given length of
    type double precision and with entries set to zero for several
    trials.
    """
    from proteus.LinearAlgebraTools import Vec
    for n in [1, 10, 100, 1000]:
        x = Vec(n)
        # Vector of length n
        assert x.size == n
        # One-dimensional
        assert x.shape == (n,)
        # Of type double-precision
        assert x.dtype == np.double
        # All entries are zero
        assert np.count_nonzero(x) == 0
        # Verify assignment works
        x[:] = list(range(1, n+1))
        assert np.count_nonzero(x) == n

@pytest.mark.LinearAlgebraTools
def test_mat_create():
    """test_mat_create

    Verifies that the proteus.LinearAlgebraTools.Mat constructor
    correctly creates double-precision matrices of the given
    dimensions and with entries set to zero for several trials.
    """

    from proteus.LinearAlgebraTools import Mat
    for m, n in [(1, 1), (1, 10), (100, 1), (500, 500)]:
        x = Mat(m, n)
        # Matrix containing m*n entries
        assert x.size == m*n
        # Two-dimensional
        assert x.shape == (m, n)
        # Of type double-precision
        assert x.dtype == np.double
        # All entries are zero
        assert np.count_nonzero(x) == 0
        # Assign a row
        x[0, :] = list(range(1, n+1))
        # Assign a column
        x[:, 0] = list(range(1, m+1))
        assert np.count_nonzero(x) == m+n-1

@pytest.mark.LinearAlgebraTools
def test_vec_scalar_math():
    """test_vec_scalar_math

    Verifies that the Vec object created by
    proteus.LinearAlgebraTools.Vec behaves as expected when computing
    basic linear algebra for several trials.
    """

    from proteus.LinearAlgebraTools import Vec

    v1 = Vec(2)
    v1[:] = [1, 2]

    npt.assert_almost_equal(v1.sum(), 3)
    npt.assert_almost_equal(v1.prod(), 2)

    v2 = Vec(2)
    v2[:] = [2, 0.5]

    # v1 + v2
    v_sum = np.asarray([3, 2.5])
    npt.assert_almost_equal(v_sum, v1+v2)

    # v1 + 2*v2
    v_scaled_sum = np.asarray([5, 3])
    npt.assert_almost_equal(v_scaled_sum, v1+2*v2)

    # 0.5*v1 + 1
    v_scalar_sum = np.asarray([1.5, 2])
    npt.assert_almost_equal(v_scalar_sum, 0.5*v1+1)

@pytest.mark.LinearAlgebraTools
def test_mat_vec_math():
    """test_mat_vec_math

    Verifies that the Mat and Vec objects from
    proteus.LinearAlgebraTools behave as expected together when
    computing basic linear algebra for one trial.
    """

    from proteus.LinearAlgebraTools import Vec
    from proteus.LinearAlgebraTools import Mat

    v1 = Vec(2)
    v1[:] = [1, 1]

    m1 = Mat(3, 2)
    m1[0, :] = [1, 2]
    m1[1, :] = [3, 2]
    m1[2, :] = [4, 5]

    # m1*v1
    dot_product = np.asarray([3, 5, 9])
    npt.assert_almost_equal(dot_product, m1.dot(v1))

@pytest.mark.LinearAlgebraTools
def test_superlu_mat():
    """test_superlu_mat

    Simple verification of functionality of SuperLU SparseMat operators.
    """

def compute_norms(h, A, vecs):
    from proteus.LinearAlgebraTools import l2Norm, l1Norm, lInfNorm, rmsNorm
    from proteus.LinearAlgebraTools import wl2Norm, wl1Norm, wlInfNorm
    from proteus.LinearAlgebraTools import energyNorm
    for norm in [l2Norm, l1Norm, lInfNorm, rmsNorm]:
        yield norm.__name__, [norm(x) for x in vecs]
    for norm in [wl2Norm, wl1Norm, wlInfNorm]:
        yield norm.__name__, [norm(x, h) for x in vecs]
    for norm in [energyNorm]:
        yield norm.__name__, [energyNorm(x, A) for x in vecs]

@pytest.mark.LinearAlgebraTools
def test_norm_correctness():
    from math import sqrt
    from proteus.LinearAlgebraTools import Mat
    from proteus.LinearAlgebraTools import l2Norm, l1Norm, lInfNorm, rmsNorm
    from proteus.LinearAlgebraTools import wl2Norm, wl1Norm, wlInfNorm
    from proteus.LinearAlgebraTools import energyNorm

    x = np.ones(2)
    h = np.ones(2)
    A = Mat(2, 2)
    A[0, 0] = 1
    A[1, 1] = 1

    assert l1Norm(x)==2
    assert l2Norm(x)==sqrt(2)
    assert lInfNorm(x)==1
    assert rmsNorm(x)==1
    assert wl1Norm(x,h)==2
    assert wl2Norm(x,h)==sqrt(2)
    assert wlInfNorm(x,h)==1
    assert energyNorm(x,A)==sqrt(2)

@pytest.mark.LinearAlgebraTools
def test_norm_zero():
    """test_norm_zero

    Norm of a zero vector better be close to zero.
    """

    from proteus.LinearAlgebraTools import Vec
    from proteus.LinearAlgebraTools import Mat

    h = Vec(2)
    h[:] = [0.5, 0.5]

    A = Mat(2, 2)
    #needs to be SPD
    A[0, :] = [5, 2]
    A[1, :] = [2, 6]

    v = Vec(2)
    v[:] = [0, 0]

    for name, norms in compute_norms(h, A, [v]):
        n = norms[0]
        assert np.allclose(n,0)

@pytest.mark.LinearAlgebraTools
def test_norm_homogeneity():
    """test_norm_homogeneity

    Test if defined norms in proteus.LinearAlgebraTools obey
    absolute homoegeneity for several trials.
    """

    from proteus.LinearAlgebraTools import Vec
    from proteus.LinearAlgebraTools import Mat

    h = Vec(2)
    h[:] = [0.5,0.5]

    A = Mat(2,2)
    #needs to be SPD
    A[0, :] = [5, 2]
    A[1, :] = [2, 6]

    v1 = Vec(2)
    v1[:] = [1, 1]

    for a in [0.5, -2]:
        for name, norms in compute_norms(h, A, [v1, a*v1]):
            t1, t2 = norms
            assert np.allclose(abs(a)*t1,t2)

@pytest.mark.LinearAlgebraTools
def test_norm_triangle_inequality():
    """test_norm_triangle_inequality

    Test if defined norms in proteus.LinearAlgebraTools obey
    the triangle inequality for several trials.
    """

    from proteus.LinearAlgebraTools import Vec
    from proteus.LinearAlgebraTools import Mat

    h = Vec(2)
    h[:] = [1.0, 1.0]

    #need an SPD matrix for this to be a norm
    A = Mat(2, 2)
    A[0, :] = [4, 2]
    A[1, :] = [2, 5]

    v1 = Vec(2)
    v1[:] = [1, 1]

    v2 = Vec(2)
    v2[:] = [-1, 4]

    for name, norms in compute_norms(h, A, [v1+v2, v1, v2]):
        t1, t2, t3 = norms
        assert t1 <= t2 + t3

@pytest.mark.LinearAlgebraTools
def test_petsc_binary_mat_io():
    """test_petsc_binary_mat_io

    Verifies that binary save/loads on PETSc Mats works correctly.
    """

    from proteus.LinearAlgebraTools import ParMat_petsc4py

    n = 10
    par_bs = 1
    par_n = n
    par_N = n
    par_nghost = 0
    subdomain2global = list(range(n))
    ghosted_csr_mat = MockMat(n)

    M = ParMat_petsc4py(ghosted_csr_mat, par_bs, par_n, par_N, par_nghost, subdomain2global)

    M.save('M.petsc_mat')

    from petsc4py import PETSc

    viewer = PETSc.Viewer().createBinary('M.petsc_mat', 'r')
    A = PETSc.Mat().load(viewer)

    mi, mj, mv = M.csr_rep
    ai, aj, av = A.getValuesCSR()
    npt.assert_equal(mi, ai)
    npt.assert_equal(mj, aj)
    npt.assert_equal(mv, av)

@pytest.mark.LinearAlgebraTools
def test_petsc_load_matrix(tmpdir):
    """test petsc_load_matrix """
    from petsc4py import PETSc as p4pyPETSc
    from proteus import LinearAlgebraTools as LAT

    vals_A =     [5.5,7.1,1.0]
    col_idx_A =  [0 , 1 , 2  ]
    row_idx_A =  [0, 1, 2, 3]
    A = LAT.csr_2_petsc(size = (3,3),
                        csr = (row_idx_A,col_idx_A,vals_A))

    A_tmp = tmpdir.join('A.petsc_mat')
    LAT._petsc_view(A,A_tmp.strpath)
    A_test = LAT.petsc_load_matrix(A_tmp.strpath)
    csr_values = A_test.getValuesCSR()

    assert np.allclose(csr_values[0], row_idx_A)
    assert np.allclose(csr_values[1], col_idx_A)
    assert np.allclose(csr_values[2], vals_A)

    A_test = LAT.petsc_load_matrix('dne.txt')
    assert A_test is None

@pytest.mark.LinearAlgebraTools
def test_petsc_load_vec(tmpdir):
    """test petsc_load_matrix """
    from petsc4py import PETSc as p4pyPETSc
    from proteus import LinearAlgebraTools as LAT

    vals_A = np.array([5.5,7.1,1.0])
    A = p4pyPETSc.Vec()
    A.createWithArray(vals_A)

    A_tmp = tmpdir.join('A.petsc_vec')
    LAT._petsc_view(A,A_tmp.strpath)
    A_test = LAT.petsc_load_vector(A_tmp.strpath)
    vec_values = A_test.getArray()

    assert np.allclose(vec_values, vals_A)

    A_test = LAT.petsc_load_vector('dne.txt')
    assert A_test is None


import enum
from typing import Any, Dict, List, Optional, Union

import attr

"""
This module holds representations for request and results for the metrics API
endpoint provided by the Snap Store.

The relevant API documentation is available at:
https://dashboard.snapcraft.io/docs/api/snap.html?#fetch-metrics-for-snaps
"""

MetricsStatus = enum.Enum(  # type: ignore
    value="MetricsStatus",
    names=[("OK", "OK"), ("FAIL", "FAIL"), ("NO DATA", "NO DATA")],
)

class MetricsNames(enum.Enum):
    DAILY_DEVICE_CHANGE = "daily_device_change"
    INSTALLED_BASE_BY_CHANNEL = "installed_base_by_channel"
    INSTALLED_BASE_BY_COUNTRY = "installed_base_by_country"
    INSTALLED_BASE_BY_OPERATING_SYSTEM = "installed_base_by_operating_system"
    INSTALLED_BASE_BY_VERSION = "installed_base_by_version"
    WEEKLY_DEVICE_CHANGE = "weekly_device_change"
    WEEKLY_INSTALLED_BASE_BY_CHANNEL = "weekly_installed_base_by_channel"
    WEEKLY_INSTALLED_BASE_BY_COUNTRY = "weekly_installed_base_by_country"
    WEEKLY_INSTALLED_BASE_BY_OPERATING_SYSTEM = (
        "weekly_installed_base_by_operating_system"
    )
    WEEKLY_INSTALLED_BASE_BY_VERSION = "weekly_installed_base_by_version"

@attr.s(auto_attribs=True)
class MetricsFilter:
    snap_id: str
    metric_name: str
    start: str
    end: str

    def marshal(self) -> Dict[str, str]:
        return attr.asdict(self)

@attr.s(auto_attribs=True)
class Series:
    name: str
    values: List[Union[str, int]]
    currently_released: Optional[bool] = None

    def marshal(self) -> Dict[str, str]:
        obj = attr.asdict(self)

        if self.currently_released is None:
            obj.pop("currently_released")

        return obj

    @classmethod
    def unmarshal(cls, payload: Dict[str, Any]) -> "Series":
        name = payload.get("name")
        if not isinstance(name, str):
            raise ValueError(f"Invalid metric name: {name!r}")

        values = payload.get("values")
        if not isinstance(values, list) or not all(
            isinstance(b, str) or isinstance(b, int) or b is None for b in values
        ):
            raise ValueError(f"Invalid metric values: {values!r}")

        currently_released = payload.get("currently_released")
        if currently_released is not None and not isinstance(currently_released, bool):
            raise ValueError(
                f"Invalid metric currently_released: {currently_released!r}"
            )

        return cls(
            name=name,
            values=values,
            currently_released=currently_released,
        )

@attr.s(auto_attribs=True)
class MetricResults:
    status: MetricsStatus
    snap_id: str
    metric_name: str
    buckets: List[str]
    series: List[Series]

    def marshal(self) -> Dict[str, Any]:
        return {
            "status": self.status.name,
            "snap_id": self.snap_id,
            "metric_name": self.metric_name,
            "buckets": self.buckets,
            "series": [s.marshal() for s in self.series],
        }

    @classmethod
    def unmarshal(cls, payload: Dict[str, Any]) -> "MetricResults":
        status_string = payload.get("status", "")
        try:
            status = MetricsStatus[status_string]
        except (KeyError, TypeError):
            raise ValueError(f"Invalid metric status: {status_string!r}")

        snap_id = payload.get("snap_id")
        if not isinstance(snap_id, str):
            raise ValueError(f"Invalid metric snap id: {snap_id!r}")

        metric_name = payload.get("metric_name")
        if not isinstance(metric_name, str):
            raise ValueError(f"Invalid metric name: {metric_name!r}")

        buckets = payload.get("buckets")
        if not isinstance(buckets, list) or any(
            [not isinstance(b, str) for b in buckets]
        ):
            raise ValueError(f"Invalid metric buckets: {buckets!r}")

        series = payload.get("series")
        if not isinstance(series, list):
            raise ValueError(f"Invalid metric series: {series!r}")

        return cls(
            status=status,
            snap_id=snap_id,
            metric_name=metric_name,
            buckets=buckets,
            series=[Series.unmarshal(s) for s in series],
        )

@attr.s(auto_attribs=True)
class MetricsResults:
    metrics: List[MetricResults]

    def marshal(self) -> Dict[str, Any]:
        return {"metrics": [m.marshal() for m in self.metrics]}

    @classmethod
    def unmarshal(cls, payload: Dict[str, Any]) -> "MetricsResults":
        if not isinstance(payload, dict):
            raise ValueError(f"Invalid metrics results: {payload!r}")

        metrics = payload.get("metrics")
        if not isinstance(metrics, list):
            raise ValueError(f"Invalid metrics: {metrics!r}")

        return cls(metrics=[MetricResults.unmarshal(m) for m in metrics])


import math, random, copy
import numpy as np
import os, sys

import torch
import torch.nn as nn
import torch.optim as optim
import torch.autograd as autograd
import torch.nn.functional as F

from DGN import DGN
from buffer_R import ReplayBuffer
from env import multi_agent_env as Surviving
from config import *

USE_CUDA = torch.cuda.is_available()
print('yes')
env = Surviving(n_agent=5)
print(env.landmarks)
n_ant = env.n_agent
observation_space = env.len_obs
n_actions = env.n_action
mask_dim = env.mask_dim

# buff = ReplayBuffer(capacity, observation_space, n_actions, n_ant)
buff = ReplayBuffer(capacity,hidden_dim, observation_space, n_actions, n_ant)
model = DGN(n_ant, observation_space, hidden_dim, n_actions,mask_dim)
model = model.cuda()
# model.load_state_dict(torch.load('model_cifar_BiGRU_hidden7070.pt'))
model.load_state_dict(torch.load('model_cifar.pt'))

best_score = 0
M_Null = torch.Tensor(np.array([np.eye(n_ant)] * batch_size)).cuda()
KL = nn.KLDivLoss()

i_episode += 1
steps = 0
obs = env.reset()
env.render()
terminated = False
hidden_states = None
epsilon = 0.0
# env.state_n= [np.array([71, 71,  0]) for _ in
#                         range(n_ant)]
    # print(adj.shape)
# while True:
#         # print(obs)
#     steps += 1
#     action = []
#     n_adj = adj + np.eye(n_ant)
#     q, a_w = model(torch.Tensor(np.array([obs])).cuda(), torch.Tensor(np.array([n_adj])).cuda())
#     q = q[0]
#     for i in range(n_ant):
#         if np.random.rand() < epsilon:
#             a = np.random.randint(n_actions)
#         else:
#             a = q[i].argmax().item()
#         action.append(a)
#
#     next_obs, next_adj, reward, terminated = env.step(action)
#     env.render()
#     # break
#     buff.add(np.array(obs), action, reward, np.array(next_obs), n_adj, next_adj, terminated)
#     obs = next_obs
#     adj = next_adj
#     score += sum(reward)
#     if terminated:
#         break

while True:
    # print(obs)
    steps += 1
    action = []
    if hidden_states is None:
        hidden_states = model.init_hidden()
    q, next_hidden_states = model(torch.Tensor(np.array([obs])).cuda(), hidden_states)
    env.render()
    # if hidden_states is not None:
    hidden_states = hidden_states.detach().cpu().numpy()
    next_hidden_states = next_hidden_states.detach().cpu().numpy()
    q = q[0]
    for i in range(n_ant):
        a = q[i].argmax().item()
        action.append(a)
    # print(action)
    next_obs, reward, terminated = env.step(action)
    # break
    buff.add(np.array(obs), action, reward, np.array(next_obs), hidden_states, next_hidden_states, terminated)
    hidden_states = torch.Tensor(next_hidden_states).cuda()
    obs = next_obs

    score += sum(reward)
    if terminated:
        break


#!/usr/bin/env python

import logging

from beepboop import resourcer
from beepboop import bot_manager

from bot.settings import config
from bot.slack_bot import SlackBot
from bot.slack_bot import spawn_bot

logger = logging.getLogger(__name__)

if __name__ == "__main__":

    log_level = config.log_level
    logging.basicConfig(format='%(asctime)s - %(levelname)s: %(message)s', level=log_level)

    slack_token = config.slack_token

    if slack_token == "":
        logging.info(
            "SLACK_TOKEN env var not set, expecting token to be provided by Resourcer events"
        )
        slack_token = None
        botManager = bot_manager.BotManager(spawn_bot)
        res = resourcer.Resourcer(botManager)
        res.start()
    else:
        # only want to run a single instance of the bot in dev mode
        bot = SlackBot(slack_token)
        bot.start({})


from tensorflow.keras.optimizers import Adam
init_lr = 1e-4
epochs = 100
opt = Adam(learning_rate=init_lr, decay=init_lr / epochs)

modelA.compile(optimizer=opt,
              loss = ["mse", # for age
                      "binary_crossentropy"], # for gender
              loss_weights = [4.,  # for age
                              0.1], # for gender
              metrics = ["mae",    # for age
                         "accuracy"]) # for gender

from keras.callbacks import ModelCheckpoint

callbacks = [
     ModelCheckpoint("./model_checkpoint", monitor='val_loss')
]

def generator_wrapper(generator):
  for batch_x,batch_y in generator:
    yield (batch_x,[batch_y[:,i] for i in range(2)])

STEP_SIZE_TRAIN=train_generator.n//train_generator.batch_size
STEP_SIZE_VALID=valid_generator.n//valid_generator.batch_size
STEP_SIZE_TEST=test_generator.n//test_generator.batch_size

history = modelA.fit_generator(
                    generator=generator_wrapper(train_generator),
                    steps_per_epoch=STEP_SIZE_TRAIN,
                    epochs = 1,
                    verbose= 2,
                    validation_data = generator_wrapper(valid_generator),
                    validation_steps=70)

model_folder = '/content/drive/MyDrive/Colab Notebooks/models'
modelA.save(model_folder+"age_gender_A.h5")


#!/usr/bin/env python3

from subprocess import call
import os
from Bio import SeqIO
import shutil
import multiprocessing
import datetime

import drep.d_filter
import drep.d_cluster

# def run_cmd(cmd,dry=False,shell=True,quiet= True):
#     devnull = open(os.devnull, 'w')
#     if shell:
#         if not dry:
#             if quiet:
#                 call(cmd,shell=True,stdout=devnull, stderr=devnull)
#             else:
#                 call(cmd,shell=True)
#         else:
#             print(cmd)
#     else:
#         if not dry:
#             if quiet:
#                 call(cmd,stdout=devnull, stderr=devnull)
#             else:
#                 call(cmd)
#         else: print(' '.join(cmd))
#     return

def run_cmd(cmd, dry=False, shell=True, logdir=False):
    # if dry, just print cmd
    if dry:
        if shell:
            print(cmd)
        else:
            print(' '.join(cmd))
        return

    # figure out what files you're going to store the output
    if logdir == False:
        devnull = open(os.devnull, 'w')
        sto = devnull
        ste = devnull
    else:
        uniq_filename = str(datetime.datetime.now().date()) + '_' + \
            str(datetime.datetime.now().time()).replace(':', '.')
        sto = open(os.path.join(logdir + uniq_filename + '.STDOUT'), 'w')
        ste = open(os.path.join(logdir + uniq_filename + '.STDERR'), 'w')

        # log the command
        now = open(os.path.join(logdir + uniq_filename + '.CMD'), 'w')
        if shell:
            now.write(str(cmd) + '\n')
        else:
            now.write(' '.join(cmd) + '\n')
        now.close()

    # run the command
    if shell:
        call(cmd,shell=True,stdout=sto, stderr=ste)
    else:
        call(cmd,stdout=sto, stderr=ste)
    return

def thread_cmd_wrapper(tup):
    run_cmd(*tup)

def thread_cmds(cmds, dry=False, shell=False, logdir=False, t=10):
    pool = multiprocessing.Pool(processes=t)
    tups = [(cmd, dry, shell, logdir) for cmd in cmds]
    pool.map(thread_cmd_wrapper, tups)
    pool.close()
    pool.join()
    return

def make_dir(outdirname,dry=False,overwrite=False):
    if dry:
        return
    if os.path.exists(outdirname):
        if overwrite:
            pass
            #shutil.rmtree(outdirname, ignore_errors=True)
            #os.makedirs(outdirname)
        else:
            assert False, "{0} already exsists! Will not overwrite with current settings".\
                            format(outdirname)
    else:
        os.makedirs(outdirname)

def clobber_dir(outdirname,dry=False,overwrite=False):
    if dry:
        return
    if os.path.exists(outdirname):
        if overwrite:
            shutil.rmtree(outdirname, ignore_errors=True)
            os.makedirs(outdirname)
        else:
            assert False, "{0} already exsists! Will not overwrite with current settings".\
                            format(outdirname)
    else:
        os.makedirs(outdirname)

def fasta_length(fasta):
    total = 0
    for seq_record in SeqIO.parse(fasta, "fasta"):
        total += len(seq_record)
    return total


import Tkinter
import time
from matplotlib.backends.backend_tkagg import FigureCanvasTkAgg,NavigationToolbar2TkAgg
from matplotlib.figure import Figure
import numpy as np
import socket
import rospy
from std_msgs.msg import String

#Visualization application for the static rtk case
class App:
	def __init__(self,master):

		#This was a final test to demonstrate that the software can be extended
		#To include the GUI directly within the positioning node
		rospy.init_node('Bamn')

		#Initialize the publisher
		self.bamn_pub = rospy.Publisher('Bamn', String, queue_size = 30)

		#The Reference Rover and Base coordinates were hard coded in order to
		#avoid "over engineering" as the intention is to eventually have this
		#information input in an "elegant" fasion
		self.ref = np.array([4641949.35634791,1393045.6421954,4133287.64031007])
		self.base = np.array([4641952.63326322,1393062.95794337,4133278.25700797])

		#Store a reference to the GUI within the obhject
		self.master = master

		#Create the network socket and bind to the "static_rtk" port
		self.sock = socket.socket()
		host = socket.gethostname()
		port = 5802
		self.sock.connect((host,port))

		#Create 2 buttons and place them at the top of the GUI. These buttons
		#change the view between baseline mode and rover mode
		buttonFrame = Tkinter.Frame(master)

		self.roverButton = Tkinter.Button(buttonFrame,text='Rover Mode',command=self.roverMode)
		self.roverButton.pack(side="left")

		self.baselineButton = Tkinter.Button(buttonFrame,text='Baseline Mode',command=self.baselineMode)
		self.baselineButton.pack(side='left')

		buttonFrame.pack()

		#Initialize the application to rover mode
		self.mode = 'rover'

		self.frame = Tkinter.Frame(master)
		self.frame.pack()

		#Create a text box to display the rover coordinates
		self.text = Tkinter.Text(master, height=5, width=20)
		output = 'x = 0\n\ny = 0\n\nz = 0'
		self.text.insert(Tkinter.END, output)
		self.text.pack(side='left')

		#Create the data structures to store the epoch-to-epoch results
		self.pos = [[],[],[]]
		self.dPos = [[],[],[]]

		self.bl = [[],[],[]]
		self.dBL = [[],[],[]]

		self.Cx = [[],[],[],[],[],[]]

		self.gpst = []
		self.ftest = []
		self.delay = []
		self.gdop = []

		#Create blank plots and store them as references within the object
		self.fig = Figure()

		self.plt = []

		for i in range(231,237):

			self.plt.append(self.fig.add_subplot(i))

		#Add the plots to the application and add the toolbar at the bottom
		#In order to interact with the plots (e.g. zoom and translate the plots)
		self.canvas = FigureCanvasTkAgg(self.fig,master=self.master)
		self.canvas.get_tk_widget().pack(side='top', fill='both', expand=1)
		self.toolbar = NavigationToolbar2TkAgg( self.canvas, master )
		self.toolbar.update()
		self.canvas._tkcanvas.pack(side=Tkinter.TOP, fill=Tkinter.BOTH, expand=1)

	#This method switches the application's mode to rover mode, which is then
	#interpreted by the rtkPlot function to plot the rover state details
	def roverMode(self):

		self.mode = 'rover'

	#This method switches the application's mode to baseline mode, which is
	#then interpreted by the rtkPlot function to plot the baseline state details
	def baselineMode(self):

		self.mode = 'baseline'

	#At each measurement epoch, get the output from RTKRCV, parse it, and use it
	#To compute the different quantities that are displayed
	def getData(self):

		#WGS84 Parameters
		e2 = 6.69437999014e-3
		a = 6378137.0

		#Receive the message from RTKRCV
		msgStr = self.sock.recv(1024)

		#Parse the message
		msg = msgStr.split()

		#Compute the radius of curvature
		N = 1.0*a/np.sqrt(1-e2*(np.sin(float(msg[2])*np.pi/180.0)**2))

		#ECEF XYZ Pose
		self.pos[0].append((N+float(msg[4]))*np.cos(float(msg[2])*np.pi/180.0)*np.cos(float(msg[3])*np.pi/180.0))
		self.pos[1].append((N+float(msg[4]))*np.cos(float(msg[2])*np.pi/180.0)*np.sin(float(msg[3])*np.pi/180.0))
		self.pos[2].append((N*(1-e2)+float(msg[4]))*np.sin(float(msg[2])*np.pi/180.0))

		#Reference Coordinate Discrepancies
		self.dPos[0].append(self.pos[0][-1] - self.ref[0])
		self.dPos[1].append(self.pos[1][-1] - self.ref[1])
		self.dPos[2].append(self.pos[2][-1] - self.ref[2])

		#Baseline and baseline discrepancy (same as reference coord discrepancy)
		self.bl[0].append(self.pos[0][-1] - self.base[0])
		self.bl[1].append(self.pos[1][-1] - self.base[1])
		self.bl[2].append(self.pos[2][-1] - self.base[2])
		self.dBL[0].append(self.bl[0][-1] - (self.ref[0] - self.base[0]))
		self.dBL[1].append(self.bl[1][-1] - (self.ref[1] - self.base[1]))
		self.dBL[2].append(self.bl[2][-1] - (self.ref[2] - self.base[2]))

		#Covariance matrix output from RTKRCV
		self.Cx[0].append(float(msg[7]))
		self.Cx[1].append(float(msg[8]))
		self.Cx[2].append(float(msg[9]))
		self.Cx[3].append(float(msg[10]))
		self.Cx[4].append(float(msg[11]))
		self.Cx[5].append(float(msg[12]))

		#Norm of the covariance matrix diagonal elements
		self.gdop.append(np.sqrt(self.Cx[0][-1]*self.Cx[0][-1] + self.Cx[1][-1]*self.Cx[1][-1] + self.Cx[2][-1]*self.Cx[2][-1]))

		#Store other GNSS relevant data
		self.gpst.append(float(msg[1]))
		self.ftest.append(float(msg[14]))
		self.delay.append(float(msg[13]))

		#Test out publishing over a rostopic without using a ROS based loop
		self.bamn_pub.publish("BAMN")

		#Call the function again after 1000 ms
		self.master.after(1000,self.getData)

	#At each measurement epoch, replot the requested quantities with the newly
	#acquired values
	def rtkPlot(self):

		#If Rover mode is selected
		if self.mode == 'rover':

			#Datasets to be plotted and their labels
			datasets = [self.dPos[0], self.dPos[1], self.dPos[2], self.gdop, self.ftest, self.delay]
			titles = ['dx', 'dy', 'dz', 'GDOP', 'ftest', 'delay']

			#Plot each dataset
			for i in range(len(datasets)):

				self.plt[i].cla()
				self.plt[i].plot(range(len(datasets[i])),datasets[i])
				self.plt[i].set_title(titles[i])

			#Display rover coordinates
			output = 'x = ' + str(self.pos[0][-1]) + '\n\ny = ' + str(self.pos[1][-1]) + '\n\nz = ' + str(self.pos[2][-1])
			self.text.delete(1.0,Tkinter.END)
			self.text.insert(Tkinter.END, output)
			self.text.pack(side='left')

		#If baseline mode is selected
		if self.mode == 'baseline':

			#Datasets to be plotted and their labels
			datasets = [self.bl[0], self.bl[1], self.bl[2], self.dBL[0], self.dBL[1], self.dBL[2]]
			titles = ['DX', 'DY', 'DZ', 'dDX', 'dDY', 'dDZ']

			#Plot each dataset
			for i in range(len(datasets)):

				self.plt[i].cla()
				self.plt[i].plot(range(len(datasets[i])),datasets[i])
				self.plt[i].set_title(titles[i])

			#Display the baseline info
			output = 'x = ' + str(self.pos[0][-1]) + '\n\ny = ' + str(self.pos[1][-1]) + '\n\nz = ' + str(self.pos[2][-1])
			self.text.delete(1.0,Tkinter.END)
			self.text.insert(Tkinter.END, output)
			self.text.pack(side='left')

		#Draw the new plots
		self.canvas.draw()

		#Call the plotting function every seconds
		self.master.after(1000,self.rtkPlot)

#Main function
root = Tkinter.Tk()
root.wm_title("RTK Data Analyzer")
app = App(root)
root.after(1000,app.getData)
root.after(1000,app.rtkPlot)
root.mainloop()


import contextlib
import collections
import copy

import advent_tools

def main():
    input_data = advent_tools.read_input_lines()
    dependencies = process_input(input_data)
    print('Part 1:', run_part_1(copy.deepcopy(dependencies)))
    print('Part 2:', run_part_2(dependencies, num_workers=5))

def process_input(data):
    result = collections.defaultdict(set)
    for line in data:
        words = line.split()
        result[words[-3]].add(words[1])
    return result

def run_part_1(dependencies):
    not_done = {val for dep in dependencies.values()
                for val in dep}.union(dependencies.keys())
    completed = []
    while not_done:
        for step in sorted(not_done):
            if not dependencies[step]:
                completed.append(step)
                not_done.remove(step)
                for dep in dependencies.values():
                    with contextlib.suppress(KeyError):
                        dep.remove(step)
                break
    return "".join(completed)

def run_part_2(dependencies, num_workers=2):
    not_started = {val for dep in dependencies.values()
                   for val in dep}.union(dependencies.keys())
    completion_times = collections.defaultdict(set)
    available = set()
    completion_times[0] = set()
    tick = 0
    while completion_times:
        tick = sorted(completion_times)[0]
        not_started = not_started.difference(completion_times[tick])
        for step, dep in dependencies.items():
            with contextlib.suppress(KeyError):
                dependencies[step] = dep.difference(completion_times[tick])
        available = available.union(
            {step for step in sorted(not_started) if not dependencies[step]}).difference(
            completion_times[tick]
        )
        del completion_times[tick]
        for worker in range(num_workers - len(completion_times)):
            if available:
                to_do = sorted(available)[0]
                available.remove(to_do)
                not_started.remove(to_do)
                completion_times[ord(to_do) - 4 + tick].add(to_do)
    return tick

if __name__ == '__main__':
    main()


from collections import deque, namedtuple
from functools import partial
from typing import Callable, Optional, Tuple, Union

import dm_env
import haiku as hk
import jax
import numpy as np
from bsuite.baselines.base import Agent
from dm_env import specs
from jax import numpy as jnp

Observation = np.ndarray
Action = int
Features = Union[np.ndarray, jnp.ndarray]
BasisFunction = Callable[[Observation, Action], Features]
AgentState = namedtuple('AgentState', "seq_weights")

@jax.jit
def _eval(
    agent_state: AgentState,
    batch_f: jnp.ndarray,
    batch_r: jnp.ndarray,
) -> AgentState:
    batch_rev_q = jnp.cumsum(batch_r[::-1]) / jnp.arange(1, 1 + len(batch_r))
    batch_q = batch_rev_q[::-1]
    weights = jnp.linalg.lstsq(batch_f, batch_q)[0]
    return agent_state._replace(seq_weights=agent_state.seq_weights +
                                [weights])

@partial(jax.jit, static_argnums=(4, ))
def _select_last(
    agent_state: jnp.ndarray,
    batch_f: jnp.ndarray,
    lr: float,
    key: np.ndarray,
    size: Optional[int] = None,
) -> Action:
    w_all = agent_state.seq_weights
    if size is not None:
        w_all = w_all[-(size + 1):]
    if len(w_all) == 1:
        q = jnp.matmul(batch_f, w_all[0])
        div = jnp.linalg.norm(q - jnp.mean(q), ord='inf')
        logits = 2 * q
        temperature = lr * jnp.sqrt(2) * div
    else:
        q_all = jnp.matmul(jnp.stack(w_all), batch_f.T)
        div = jnp.linalg.norm(q_all[1:] - q_all[:-1], ord='inf', axis=1)
        logits = q_all[1:].sum(axis=0) + q_all[-1]
        temperature = lr * jnp.sqrt(2 * jnp.power(div, 2).sum())
    action = jax.random.categorical(key, logits / temperature)
    return action

@partial(jax.jit, static_argnums=(4, ))
def _select_random(
    agent_state: jnp.ndarray,
    batch_f: jnp.ndarray,
    lr: float,
    key: np.ndarray,
    size: Optional[int] = None,
) -> Action:
    w_all = agent_state.seq_weights
    q = jnp.matmul(batch_f, w_all[-1])
    if (size is not None) and len(w_all) > size:
        idxs = jax.random.choice(key,
                                 jnp.arange(1, len(w_all)), (size, ),
                                 replace=False)
        scale = jnp.sqrt(len(w_all) / size)
    else:
        idxs = jnp.arange(1, len(w_all))
        scale = 1
    if len(w_all) == 1:
        div = jnp.linalg.norm(q - jnp.mean(q), ord='inf')
        logits = 2 * q
        temperature = lr * jnp.sqrt(2) * div
    else:
        w = jnp.stack(w_all)[idxs]
        w_ = jnp.stack(w_all)[idxs - 1]
        q_all = jnp.matmul(jnp.stack(w), batch_f.T)
        q_all_ = jnp.matmul(jnp.stack(w_), batch_f.T)
        div = jnp.linalg.norm(q_all - q_all_, ord='inf', axis=1)
        logits = q_all.sum(axis=0) + q
        temperature = lr * jnp.sqrt(2 * jnp.power(div, 2).sum())
        temperature *= scale
    action = jax.random.categorical(key, logits / temperature)
    return action

class Buffer:
    def __init__(self, capacity: Optional[int] = None):

        self._observations = deque(maxlen=capacity)
        self._actions = deque(maxlen=capacity)
        self._rewards = deque(maxlen=capacity)

    def append(self, timestep: dm_env.TimeStep, action: Action,
               new_timestep: dm_env.TimeStep):

        self._observations.append(timestep.observation)
        self._actions.append(action)
        self._rewards.append(new_timestep.reward)

        if new_timestep.last():
            raise AssertionError(
                'Environment should have infinite horizon (see paper).')

    def get(self) -> Tuple[np.ndarray, np.ndarray, np.ndarray]:
        batch_o = np.stack(self._observations)
        batch_a = np.stack(self._actions)
        batch_r = np.stack(self._rewards)
        return batch_o, batch_a, batch_r

class AAPI(Agent):
    def __init__(self,
                 obs_spec: specs.Array,
                 action_spec: specs.DiscreteArray,
                 basis_function: BasisFunction,
                 train_every: int,
                 learning_rate: float,
                 buffer_size: int,
                 rng: hk.PRNGSequence,
                 weight_select: str = 'last',
                 weight_size: Optional[int] = None):
        assert weight_select in [
            'random', 'last'
        ], "'weight_select' take values in ['random', 'last'] "
        self._obs_spec = obs_spec
        self._action_spec = action_spec
        self._basis_function = jax.vmap(basis_function)
        self._train_every = train_every
        self._learning_rate = learning_rate
        self._rng = rng
        self._weight_size = weight_size

        if weight_select == 'random':
            self._select = _select_random
        else:
            self._select = _select_last

        # initialize buffer
        capacity = buffer_size if buffer_size > 0 else None
        self._t = 0
        self._buffer = Buffer(capacity)

        # initialize parameters
        self._state = AgentState(seq_weights=[])

    def update(
        self,
        timestep: dm_env.TimeStep,
        action: Action,
        new_timestep: dm_env.TimeStep,
    ):
        self._t += 1
        self._buffer.append(timestep, action, new_timestep)
        if self._t % self._train_every == 0:
            batch_o, batch_a, batch_r = self._buffer.get()
            batch_f = self._basis_function(batch_o, batch_a)
            self._state = _eval(self._state, batch_f, batch_r)

    def select_action(
        self,
        timestep: dm_env.TimeStep,
    ) -> Action:
        nA = self._action_spec.num_values
        if len(self._state.seq_weights) == 0:
            action = jax.random.randint(next(self._rng), (), 0, nA)
            return int(action)
        lr = self._learning_rate
        o = timestep.observation
        batch_o = jnp.repeat(np.expand_dims(o, 0), nA, axis=0)
        batch_a = jnp.arange(nA)
        batch_f = self._basis_function(batch_o, batch_a)
        action = self._select(self._state, batch_f, lr, next(self._rng),
                              self._weight_size)
        return int(action)


#!/usr/bin/env python

from setuptools import setup, find_packages

version = '0.1dev'

with open('README.md') as f:
    readme = f.read()

with open('LICENSE') as f:
    license = f.read()

with open('requirements.txt') as f:
    required = f.read().splitlines()

setup(
    name='carlosmatos',
    version=version,
    description='Parser for simple Graph structure',
    long_description=readme,
    keywords=['test'],
    author='Philipp Ehmele',
    author_email='philipp_ehm@protonmail.com',
    license=license,
    scripts=['scripts/carlosmatos'],
    install_requires=required,
    packages=find_packages(exclude='docs'),
    include_package_data=True
)


from ops import launch

def count_participants(sessionsCount, sessionCapacity, teams):
    people = sum(teams)

    memo = [None] * (sessionCapacity + 1)
    participants = 0
    sessionsDone = 0

    waitingForSession = 0
    while True:
        if memo[waitingForSession] is not None and sessionsDone - memo[waitingForSession][1] <= sessionsCount - sessionsDone:
            cycleAddedPeople = participants - memo[waitingForSession][0]
            cycleDoneSessions = sessionsDone - memo[waitingForSession][1]
            repeats = (sessionsCount - sessionsDone) // cycleDoneSessions

            participants += cycleAddedPeople * repeats
            sessionsDone += cycleDoneSessions * repeats

            if sessionsDone == sessionsCount:
                return participants
        else:
            participantsBefore = participants
            waitingForSessionBefore = waitingForSession
            sessionsDoneBefore = sessionsDone

            for team in teams:
                if waitingForSession + team > sessionCapacity:
                    sessionsDone += 1
                    participants += waitingForSession
                    waitingForSession = 0
                    if sessionsDone == sessionsCount:
                        return participants
                waitingForSession += team
            if sessionsDoneBefore == sessionsDone or waitingForSession == sessionCapacity:
                sessionsDone += 1
                participants += waitingForSession
                waitingForSession = 0
                if sessionsDone == sessionsCount:
                    return participants
            memo[waitingForSessionBefore] = (participantsBefore, sessionsDoneBefore)

def task(input, output):
    R, K, N = [int(x) for x in input.readline().split(' ')]
    assert N > 0 and R > 0 and K > 0
    teams = [int(x) for x in input.readline().split(' ')]
    assert len(teams) == N
    output.write(str(count_participants(R, K, teams)) + '\n')
    print('Done')

launch(task, 'G-lehky.txt')


# reqDataInsertion.py
# Created by Nate O, modified by Mickey G for Project flextape use
# To Import Req data to mongodb, within the flextape pipeline

import pandas as pd
import pymongo
from pymongo import MongoClient
from tqdm import tqdm
import sys
import time
import os

print("reqDataInsertion.py taking over...")

if not sys.argv[1] and not sys.argv[2]:
    raise ValueError('Arguments Needed: Date String (mmddyyyy-hhmmss), Write Location: One of (dev, prod, both)')
date = sys.argv[1]
writelocation = ['dev', 'prod'] if sys.argv[2] == 'both' else [sys.argv[2]]

serverlocation = os.environ['RUBIXLOCATION']

if serverlocation == 'local':
    filepathprefix = "/home/rubix/Desktop/Project-Ducttape/data/"
elif serverlocation == 'ohio':
    filepathprefix = "/home/ubuntu/Projects/flextape/"
else:
    raise EnvironmentError('Environment Variable "RUBIXLOCATION" seems not to be set.')

filepath = filepathprefix + "req-6/" + date + "/PLIT_REQ_6-" + date + ".xlsx"

print("--- Reading " + filepath + " ---")

insertionItems = pd.read_excel(filepath, skiprows = 1)
insertionItems['Due'] = insertionItems['Due'].fillna(pd.Timestamp('2001-01-01 00:00:00'))

client = MongoClient()
insertionItems.columns = [c.replace(' ', '_') for c in insertionItems.columns]
insertionItems.columns = [c.replace('/', '_') for c in insertionItems.columns]

print("--- Pushing Data to mongo ---")

print(list(insertionItems))

print("Removing Entries to be Updated...")

print("Adding Data...")

na_values = {"Address_2": ""}
insertionItems = insertionItems.fillna(value = na_values)

for location in writelocation:
    uniqueReqIDs = dict((reqno, False) for reqno in insertionItems['Req_ID'].unique().tolist())
    uniqueItemArr = insertionItems['Item'].unique().tolist()
    dbname = 'rubix-' + serverlocation + '-' + location
    print('Using database ' + dbname)
    db = client[dbname]
    for row in tqdm(insertionItems.itertuples()):
        if not uniqueReqIDs[row.Req_ID]:
            db.REQ_DATA.update({'REQ_No': row.Req_ID}, {
                '$set': {
                    'REQ_No': row.Req_ID,
                    'Account': row.Account,
                    'Business_Unit': row.Unit,
                    'Buyer': row.Buyer,
                    'Currency' : row.Currency,
                    'Department' : {
                        "Number": row.Dept_Loc
                    },
                    "Fund": row.Fund,
                    "Origin": row.Origin,
                    "REQ_Date": row.Req_Date,
                    "Requester": row.Requester,
                    "Ship_To": {
                        "Description": row.Descr,
                        "Address_1": row.Address_1,
                        "Address_2": row.Address_2,
                        "City": row.City,
                        "State": row.St,
                        "Zip_Code": row.Postal,
                        "Country": row.Cntry
                    },
                    "Status": row.Status,
                    "Approved_By" : row.By,
                    "Approved_On": row.Date,
                    "Vendor": {
                        "Number": row.Vendor,
                        "Name": row.Name
                    },
                    "lines": []
            }
            }, upsert = True)
            uniqueReqIDs[row.Req_ID] = True

    for row in tqdm(insertionItems.itertuples()):
        db.REQ_DATA.update({"REQ_No": row.Req_ID}, {
            '$addToSet': {
                "lines": {
                        "Line_No": row.Line,
                        "Unit_Price": row.Base_Price,
                        "Line_Total": row.Amount,
                        "Schedule_No": row.Sched_Num,
                        "Quantity": row.Req_Qty,
                        "Due_Date": row.Due,
                        "More_Info": row.More_Info,
                        "UOM": row.UOM,
                        "Item": row.Item
                    }
            }})
        # db.REQ_DATA.find({"lines": {$elemMatch: {"Item": "02545903"}}})

    db.LAST_UPDATED.update({'dbname': "REQ_DATA"}, {'$set': {'last_updated_time': time.time()}})

    print("Req Update Done!")


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
@author: karanrochlani
"""

from PIL import Image
import os
import numpy as np

set1=[]
number=[]

for i in range(0,43):
    s=str(i)
    cur_path=os.path.join('/Users/karanrochlani/Desktop/Traffic-Sign-Classification/untitled folder 2/traffic sign recognition/Train',s)
    images = os.listdir(cur_path)
    for a in images:
            try:
                path=os.path.join(cur_path,a)
                image = Image.open(path)
                image = image.resize((30,30))
                image = np.array(image)
                set1.append(image)
                number.append(1)
            except:
                print("Error")

set1 = np.array(set1)
number = np.array(number)

from sklearn.model_selection import train_test_split
X_train, X_test, y_train, y_test = train_test_split(set1, number, test_size=0.2, random_state=42)

from keras.utils import to_categorical
y_train = to_categorical(y_train, 43)
y_test = to_categorical(y_test, 43)

from keras.models import Sequential
from keras.layers import Conv2D
from keras.layers import MaxPool2D
from keras.layers import Flatten
from keras.layers import Dense

classifier = Sequential()
classifier.add(Conv2D(filters=32, kernel_size=(5,5), activation='relu', input_shape=X_train.shape[1:]))
classifier.add(Conv2D(filters=32, kernel_size=(5,5), activation='relu'))
classifier.add(MaxPool2D(pool_size=(2, 2)))

classifier.add(Conv2D(filters=64, kernel_size=(3, 3), activation='relu'))
classifier.add(Conv2D(filters=64, kernel_size=(3, 3), activation='relu'))
classifier.add(MaxPool2D(pool_size=(2, 2)))

classifier.add(Flatten())

classifier.add(Dense(256, activation='relu'))

classifier.add(Dense(43, activation='sigmoid'))

classifier.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])

classifier.fit(X_train, y_train, batch_size=32, epochs=2, validation_data=(X_test, y_test))


import asyncio
from asyncio import Transport, Future, AbstractEventLoop
from typing import Optional

class HTTPGetClientProtocol(asyncio.Protocol):

    def __init__(self, host: str, loop: AbstractEventLoop):
        self._host: str = host
        self._future: Future = loop.create_future()
        self._transport: Optional[Transport] = None
        self._response_buffer: bytes = b''

    async def get_response(self):  # A
        return await self._future

    def _get_request_bytes(self) -> bytes:  # B
        request = f"GET / HTTP/1.1\r\n" \
                  f"Connection: close\r\n" \
                  f"Host: {self._host}\r\n\r\n"
        return request.encode()

    def connection_made(self, transport: Transport):
        print(f'Connection made to {self._host}')
        self._transport = transport
        self._transport.write(self._get_request_bytes())  # C

    def data_received(self, data):
        print(f'Data received!')
        self._response_buffer = self._response_buffer + data  # D

    def eof_received(self) -> Optional[bool]:
        self._future.set_result(self._response_buffer.decode())  # E
        return False

    def connection_lost(self, exc: Optional[Exception]) -> None:  # F
        if exc is None:
            print('Connection closed without error.')
        else:
            self._future.set_exception(exc)


import dash_core_components as dcc
import plotly.express as px
from dash import Input, Output
from colour_constants import custom_colors

# Layout for the bar graph
bargraph_layout_boolean = dcc.Graph(
    id="graph-boolean",
    config={"responsive": True},
    style={'width': '100%', 'height': '300px'},
)

def bargraph_callback_boolean(app, data, default_processors):
    @app.callback(
        Output("graph-boolean", "figure"),
        Input("cpu-processor-dropdown", "value")
    )
    def update_bar_chart_boolean(selected_processors):
        if not selected_processors:
            selected_processors = default_processors
        elif not isinstance(selected_processors, list):
            selected_processors = [selected_processors]

        # Selecting boolean columns for analysis
        boolean_columns = ['has_touchscreen', 'keyboard_backlit', 'has_webcam', 'has_bluetooth']

        # Group by CPU processor and count occurrences of boolean values
        boolean_stats_df = data.groupby('cpu_processor')[boolean_columns].sum().reset_index()

        # Filter the data based on selected processors
        filtered_df = boolean_stats_df[boolean_stats_df['cpu_processor'].isin(selected_processors)]

        fig = px.bar(
            filtered_df,
            x="cpu_processor",
            y=boolean_columns,
            barmode='group',
            title="Boolean Feature Counts by CPU Processor",
            labels={'value': 'Count'},
        )
        fig.update_xaxes(title_text="CPU Processor")
        fig.update_yaxes(title_text="Count")
        fig.update_layout(
            plot_bgcolor=custom_colors['background'],
            paper_bgcolor=custom_colors['background'],
            font_color=custom_colors['text'],
            font_family="FreeSerif",
            title_x=0.5,
            title_y=0.95,
            title_xanchor='center',
            title_yanchor='top',
            margin=dict(l=20, r=20, t=60, b=20),
        )

        return fig


from matplotlib import pyplot as plt
import pandas as pd
from src.cluster import KMeans

df = pd.read_csv("data/assignment-example.csv", index_col="id")
sse = []
for k in range(1, 9):
    model = KMeans(k=k, verbose=True)
    model.fit(df)
    res = [model.predict(i) for i in df.values]
    s = [model.distance_fn(i, model.centers[r]) for i, r in zip(df.values, res)]
    print(f"Sum of squared errors: {sum(s)}")
    sse.append(sum(s))
plt.plot(range(1, 9), sse)
plt.xlabel("k")
plt.ylabel("SSE")
plt.show()
print(f"Centers: {model.centers}")


#!/usr/bin/env python3

import unittest
import ipaddress
from framework import VppTestCase, VppTestRunner

from scapy.layers.inet6 import IPv6, ICMPv6EchoRequest
from scapy.layers.l2 import Ether
from scapy.packet import Raw

class TestNPT66(VppTestCase):
    """NPTv6 Test Case"""

    def setUp(self):
        super(TestNPT66, self).setUp()

        # create 2 pg interfaces
        self.create_pg_interfaces(range(2))

        for i in self.pg_interfaces:
            i.admin_up()
            i.config_ip6()
            i.resolve_ndp()

    def tearDown(self):
        for i in self.pg_interfaces:
            i.unconfig_ip6()
            i.admin_down()
        super(TestNPT66, self).tearDown()

    def send_and_verify(self, in2out, internal, external):
        if in2out:
            sendif = self.pg0
            recvif = self.pg1
            local_mac = self.pg0.local_mac
            remote_mac = self.pg0.remote_mac
            src = ipaddress.ip_interface(internal).ip + 1
            dst = self.pg1.remote_ip6
        else:
            sendif = self.pg1
            recvif = self.pg0
            local_mac = self.pg1.local_mac
            remote_mac = self.pg1.remote_mac
            src = self.pg1.remote_ip6
            dst = ipaddress.ip_interface(external).ip + 1

        p = (
            Ether(dst=local_mac, src=remote_mac)
            / IPv6(src=src, dst=dst)
            / ICMPv6EchoRequest()
        )
        rxs = self.send_and_expect(sendif, p, recvif)
        for rx in rxs:
            rx.show2()
            original_cksum = rx[ICMPv6EchoRequest].cksum
            del rx[ICMPv6EchoRequest].cksum
            rx = rx.__class__(bytes(rx))
            self.assertEqual(original_cksum, rx[ICMPv6EchoRequest].cksum)

    def do_test(self, internal, external):
        self.vapi.npt66_binding_add_del(
            sw_if_index=self.pg1.sw_if_index,
            internal=internal,
            external=external,
            is_add=True,
        )
        self.vapi.cli(f"ip route add {internal} via {self.pg0.remote_ip6}")

        self.send_and_verify(True, internal, external)
        self.send_and_verify(False, internal, external)

        self.vapi.npt66_binding_add_del(
            sw_if_index=self.pg1.sw_if_index,
            internal=internal,
            external=external,
            is_add=False,
        )

    def test_npt66_simple(self):
        """Send and receive a packet through NPT66"""

        self.do_test("fc00:1::/48", "2001:db8:1::/48")
        self.do_test("fc00:1234::/32", "2001:db8:1::/32")
        self.do_test("fc00:1234::/63", "2001:db8:1::/56")

if __name__ == "__main__":
    unittest.main(testRunner=VppTestRunner)


import nltk
from nltk.chat.util import Chat, reflections

pairs = [
    [
        r"my name is (.*)",
        ["Hello %1, How are you today?"]
    ],
    [
        r"hi|hello",
        ["Hello", "Hi there"]
    ],
    [
        r"what is your name?",
        ["My name is Chatbot. Nice to meet you!"]
    ],
    [
        r"how are you?",
        ["I'm doing well. Thanks for asking."]
    ],
    [
        r"bye",
        ["Bye!", "Goodbye!"]
    ],
]

def chatbot():
    print("Hello, I'm Chatbot! How can I help you today?")

    chat = Chat(pairs, reflections)
    chat.converse()

if __name__ == "__main__":
    nltk.download('punkt')
    chatbot()


import ui.lang_buttons as lb
from utils.utils import log_err

def _get_slice(data, page, slice=8):
    start = page*slice
    end = start + slice
    try:
        return data[start:end]
    except IndexError:
        return data[start:]

def _get_buttons(data, callback, columns=3):
    keyboard = {'inline_keyboard': []}
    if data and callback:
        page = 0
        while True:
            slices = _get_slice(data, page, slice=columns)
            if slices:
                row = [{'text': x[0], 'callback_data': f'{callback}-{x[1]}'} for x in slices]
                keyboard['inline_keyboard'].append(row)
                page += 1
            else:
                break
    return keyboard

def build_page_keyboard(data,
                        page,
                        data_callback,
                        menu_callback,
                        back_callback=None,
                        none_button=False,
                        lang=None):
    if page is None:
        page = 0
    keyboard = _get_buttons(_get_slice(data, page),
                            data_callback)
    control_buttons = []
    if page > 0:
        back_button = {'text': lb.back[lang],
                       'callback_data': f'{menu_callback}/{page - 1}-'}
        control_buttons.append(back_button)
    next_data = _get_slice(data, page + 1)
    if next_data:
        next_button = {'text': lb.next_[lang],
                       'callback_data': f'{menu_callback}/{page + 1}-'}
        control_buttons.append(next_button)
    keyboard['inline_keyboard'].append(control_buttons)
    if none_button:
        keyboard['inline_keyboard'].append([{'text': lb.doesnt_matter[lang],
                                             'callback_data': f'{data_callback}-0'}])
    if back_callback:
        keyboard['inline_keyboard'].append([{'text': lb.back[lang],
                                             'callback_data': f'{back_callback}-'}])
    return keyboard

# [[text, callback_data], ...]
def build_buttons(data):
    return [{'text': x[0], 'callback_data': x[1]} for x in data]

def build_common_keyboard(data,
                          data_callback,
                          buttons=None,
                          in_row=True):
    keyboard = _get_buttons(data, data_callback)
    if buttons:
        buttons_ = build_buttons(buttons)
        if in_row:
            keyboard['inline_keyboard'].append(buttons_)
        else:
            for b in buttons_:
                keyboard['inline_keyboard'].append([b])
    return keyboard

def search_bd(q, data):
    if q:
        for i in data:
            if i[1] == q:
                return i[0]

def normalize_text(input_string):
    if isinstance(input_string, str):
        return input_string.replace('\\n', '\n').replace('`', '')

def get_parameter(text : str, parameter):
    if parameter:
        return text % parameter
    else:
        return ''

async def translate_item(translator, lang, title, description):
    if title and description:
        text = f'{title} // {description}'
        try:
            translated = await translator.translate(text, dest=lang)
            title, description = str(translated.text).split('//')
            return title, description
        except Exception as err:
            log_err(err)
            return title, description


import re
import pandas as pd
import requests
from bs4 import BeautifulSoup

SPECIFIC_SYMBOLS = "!\"#$%&'()*+,-./:;<=>?@[]^_`{|}~"

WORDS_TO_FIND = ["sofa", "compressor", "fryer", "hose", "bookcase", "bookshelves",
                 "cushion", "lamp", "pump", "dishwasher", "shelves",
                 "stairs", "mattress", "rug", "utensil", "appliance", "dresser", "drawer",
                 "mirror", "jug", "pot", "pillow", "accessories", "sofa", "vase",
                 "blanket", "chair", "furniture", "table", "decor", "mats"]

# Remove leading and trailing spaces and replace multiple spaces with a single space
def correct_spaces(word):
    return re.sub(r'\s+', ' ', word.strip())

# Web scraping function
def scrape_page(url):
    """
      Scrape a web page and extract its text content.

      Parameters:
      - url (str): The URL of the web page to scrape.

      Returns:
      - str or None: Text content of the web page if successful, None otherwise.
      """
    try:
        response = requests.get(url)
        if response.status_code == 200:
            soup = BeautifulSoup(response.content, 'html.parser')
            text_content = soup.get_text()
            return text_content
        else:
            print(f"Failed to fetch {url}. Status code: {response.status_code}")
    except Exception as e:
        print(f"An error occurred: {e}")

def quick_scrape_page(url):
    """
       Perform a quick scrape of a web page to extract text content.
       Ignores exeptions

       Parameters:
       - url (str): The URL of the web page to scrape.

       Returns:
       - str: Cleaned text content extracted from the web page.
       """
    text_content = ""
    try:
        response = requests.get(url)
        if response.status_code == 200:
            soup = BeautifulSoup(response.content, 'html.parser')
            text_content = soup.get_text()
    except requests.RequestException:
        pass
    finally:
        if text_content != "":
            text_content = clean_input_content(text_content)
        return text_content

def collect_data(urls):
    text = ""
    for url in urls:
        if (url.startswith("http")):
            text_content = scrape_page(url)
            if text_content:
                text += " " + text_content
        else:
            print(f"{url} is not URL")
    return text

def clean_text(text):
    # remove specific symbols
    cleaned_text = re.sub(f"[{re.escape(SPECIFIC_SYMBOLS)}]", "", text)
    # remove numbers
    cleaned_text = re.sub(r'\d+', '', cleaned_text)
    # remove extra new lines
    cleaned_text = '\n'.join(sorted(set(line.strip().lower() for line in cleaned_text.split('\n'))))
    cleaned_data = []
    for word in cleaned_text.split('\n'):
        # remove non alphabet symbols, non relevant words, and country names, save in lower case
        if (all(ord(char) < 128 for char in word) and len(word) > 2):
            cleaned_data.append(correct_spaces(word))
    # remove duplicates and sort it, then return
    cleaned_data = sorted(set(cleaned_data))
    return cleaned_data

def collect_sentences_with_keywords(sentences, keywords):
    collected_sentences = []
    for sentence in sentences:
        if any(keyword.lower() in sentence.lower() for keyword in keywords):
            collected_sentences.append(sentence)

    return collected_sentences

def find_word_indexes_in_sentences(sentences, words_to_find):
    result = []
    for sentence in sentences:
        sentence_result = {"sentence": sentence, "entities": []}

        for word in words_to_find:
            word_indexes = []
            start_index = sentence.lower().find(word.lower())

            while start_index != -1:
                end_index = start_index + len(word)  # - 1
                word_indexes.append((start_index, end_index, 'PRODUCT'))
                start_index = sentence.lower().find(word.lower(), start_index + 1)

            if word_indexes:
                sentence_result["entities"].extend(word_indexes)

        if sentence_result["entities"]:
            result.append((sentence_result["sentence"], {"entities": sentence_result["entities"]}))

    return result

def get_train_data(path):
    df = pd.read_csv(path, header=None)
    sentences = df.iloc[:, 0].tolist()
    train_data = []
    result = find_word_indexes_in_sentences(cut_sentence(sentences), WORDS_TO_FIND)
    for sentence, entities in result:
        train_data.append((sentence, entities))

    return train_data

def collect_and_save_data(urlCsvPath):
    df = pd.read_csv(urlCsvPath)
    urls = df["url"].tolist()

    text = collect_data(urls)
    cleaned_data = clean_text(text)
    collected_sentences = collect_sentences_with_keywords(cleaned_data, WORDS_TO_FIND)
    # print(collected_sentences)
    df = pd.DataFrame(collected_sentences)
    df.to_csv("training_data.csv", header=False, index=False)
    print("Collect and save done.")

# cut target sentence if it is longer than 255
def cut_sentence(sentence, max_length=255):
    if len(sentence) <= max_length:
        return sentence
    else:
        return sentence[:max_length]

# cut target sentence if it
def clean_input_content(raw_content):
    # print(raw_content)
    # remove specific symbols
    cleaned_text = re.sub(f"[{re.escape(SPECIFIC_SYMBOLS)}]", "", raw_content)
    # remove numbers
    cleaned_text = re.sub(r'\d+', '', cleaned_text)
    # remove extra new lines
    cleaned_text = ' '.join(sorted(set(line.strip().lower() for line in cleaned_text.split('\n'))))

    # make it smaller texts to predict
    words = cleaned_text.split()
    ten_word_list = [' '.join(words[i:i + 10]) for i in range(0, len(words) - 9, 10)]
    return ten_word_list

# read urls from given csv file and ignore non url strings
def load_targat_urls(csv_file_path):
    df = pd.read_csv(csv_file_path, header=None)
    urls = df.iloc[:, 0].tolist()
    urls = [url for url in urls if url.startswith('https://')]
    return urls


from PIL import Image
from PIL import ImageDraw
from PIL import ImageChops

import quad_coll
import os

depths = [4, 6, 8, 12, 14, 16]
sizes = [2, 4, 6, 8, 10]
ratios = [.01, .05, .1, .25]
opacities = [.1, .25, .5, .75, .9]

for f in os.listdir('.'):
    try:
        img = Image.open(f)
    except IOError:
        pass
    else:
        idx = f.rfind('.')
        if (idx == -1):
            imgName = f
            imgExtension = ''
        else:
            imgName = f[:idx]
            imgExtension = f[idx:]

        dirName = imgName + '_test'
        try:
            os.mkdir(dirName)
        except OSError as e:
            if (e.errno != 17):
                raise e

        n = 0
        csv = file(dirName + '/' + imgName + '_trees.csv', 'w')
        csv.write('Input file name,Output file name,Max tree depth,Min node size,Min fill ratio,Min pixel opacity,# of nodes,# of leaf nodes,Image pixels,Tree pixels,False positives,False negatives,Total error\n')

        for d in depths:
            quad_coll.MAX_DEPTH = d
            for s in sizes:
                quad_coll.MIN_SIZE = s
                for r in ratios:
                    quad_coll.MIN_RATIO = r
                    for o in opacities:
                        quad_coll.MIN_OPACITY = o

                        tree = quad_coll.makeTree(img)

                        img2 = Image.new('RGBA', img.size)
                        quad_coll.drawTree(img2, tree, True)
                        draw = ImageDraw.Draw(img2)
                        draw.text((0, 0), 'max tree depth: %d' % d)
                        draw.text((0, 10), 'min node size: %d pixels' % s)
                        draw.text((0, 20), 'min node fill ratio: %f pixels' % r)
                        draw.text((0, 30), 'min filled pixel opactiy:%f alpha' % o)

                        outfilename = dirName + '/' + imgName + ('%04d' % n) + f[f.rfind('.'):]
                        ImageChops.composite(img2, img, img2).save(outfilename)
                        n = n + 1

                        error = quad_coll.calcError(img, tree)
                        csv.write('%s,%s,%d,%d,%f,%f,%d,%d,%d,%d,%d,%d,%d\n' % (\
                            f,\
                            outfilename,\
                            d,\
                            s,\
                            r,\
                            o,\
                            tree.getNumNodes(),\
                            tree.getNumLeafNodes(),\
                            error['imgPixels'],\
                            error['treePixels'],\
                            error['falsePositives'],\
                            error['falseNegatives'],\
                            error['falsePositives'] + error['falseNegatives']\
                        ))

        csv.close()


from flask import Flask, render_template, request, jsonify
import yaml
from src.models.predict_model import predict_value

def retrieve_params(config):
    with open(config) as yaml_file:
        params = yaml.safe_load(yaml_file)
    return params

params = retrieve_params(config="params.yaml")

app = Flask(
    __name__,
    static_folder=params["webapp"]["static_folder"],
    template_folder=params["webapp"]["template_folder"],
)

@app.route("/", methods=["GET", "POST"])
def home():
    if request.method == "GET":
        return render_template("home.html")
    else:
        if request.form:
            response_ = predict_value(request.form)
            return render_template("home.html", response_=response_)
        elif request.json:
            response_ = predict_value(request.json)
            return jsonify(response_)

if __name__ == "__main__":
    app.run(debug=True)


import os
import glob
from tqdm import tqdm
import time
from PIL import Image, ImageOps

base_path = "E:/Python/ImageNet/imagenet_2"

def hoge():
    train_dir = os.path.join(base_path, "train")
    train_classes = []
    for subdir in sorted(os.listdir(train_dir)):
        if os.path.isdir(os.path.join(train_dir, subdir)):
            train_classes.append(subdir)

    val_dir = os.path.join(base_path, "val")
    val_classes = []
    for subdir in sorted(os.listdir(val_dir)):
        if os.path.isdir(os.path.join(val_dir, subdir)):
            val_classes.append(subdir)

    flags = [x == y for x, y in zip(train_classes, val_classes)]
    print(all(flags))
    print(len(train_classes))
    print(len(val_classes))

def hoge():
    for file in sorted(os.walk(base_path+"/train/n01440764")):
        print(file)

def load_time():
    root_dir_hdd = base_path+"/train/n01440764/*"
    base_path_ssd = "G:/n01440764/*"
    files = sorted(glob.glob(base_path_ssd))
    start_time = time.time()
    for i in tqdm(range(10)):
        for f in files:
            with Image.open(f) as img:
                #img = ImageOps.autocontrast(img, i*0.1)
                img = img.resize((256, 256), Image.LINEAR)
                #img = ImageOps.autocontrast(img.resize((256, 256), Image.LINEAR), i*0.1)

                # HDD
                # LANCZOS -> 17.33s, 9.83s
                # autocon 11.33s, 10.99s
                # LANCZOS+autocon -> 15.02s, 14.99s
                # LINEAR+autocon -> 13.70, 13.07s

                # SSD
                # LINEAR -> 7.42s 7.49s 12分
                # LANCZOS -> 9.31s 9.31s
                # autocon -> 10.82s 11.04s
                # LANCZOS+autocon -> 14.85s 15.07s
                # LINEAR+autocon -> 13.01s 13.01s 22分

    print(time.time()-start_time)

import tensorflow as tf
import numpy as np

def test():
    root_dir_hdd = base_path+"/train/n01440764/*"
    base_path_ssd = "G:/n01440764/*"
    files = sorted(glob.glob(base_path_ssd))
    start_time = time.time()
    for i in tqdm(range(10)):
        for f in files:
            with Image.open(f) as img:
                array = np.expand_dims(np.asarray(img, np.uint8), 0)
                convert = tf.image.resize_bicubic(array, (256, 256))

test()