import os
import numpy as np
import matplotlib.pyplot as plt

# x = 1
# y = 1
step  = 0.03
# xs = np.arange(-2.2, 0.5, step=step)
# ys = np.arange(-1.2, 1.2, step=step)
xs = np.arange(-1.5, 1.5, step=step)
ys = np.arange(-1.5, 1.5, step=step)
z_threshold = 2
n_threshold = 30
zs = []
cs = []
ns = []
plot_xs = []
plot_ys = []
cx = 0.5
cy = 0.1
c = cx + 1j * cy
for x in xs:
    for y in ys:
        # print("x: {}, y: {}".format(x, y))
        # c = x + 1j * y
        # z = c
        z = x + 1j * y
        n = 0
        while(True):
            z = z*z + c
            if (abs(z) > z_threshold or n > n_threshold):
#                 print(n)
                break
            n = n + 1
            # print("z: {}, n: {}".format(z, n))
            # if n > 5:
            #     raise Exception
        zs.append(z.real)
        plot_xs.append(x)
        plot_ys.append(y)
        cs.append(c.real)
        ns.append(n)

# julia
plt.xlabel('x')
plt.ylabel('y')
plt.title('Julia set c={} + {}i'.format(cx, cy))
cmap = 'Reds'
plt.scatter(plot_xs, plot_ys, c=zs, cmap=cmap, edgecolors='none', alpha=0.4)
plt.show()
# os.mkdir('imgs/julia')
# plt.savefig('imgs/julia/julia_{}_{}.png'.format(cx, cy))

# plt.xlabel('c')
# plt.ylabel('z')
# plt.scatter(cs, zs, c=ns, cmap="Reds")
# plt.show()


# uncompyle6 version 3.7.4
# Python bytecode 2.4 (62061)
# Decompiled from: Python 3.6.9 (default, Apr 18 2020, 01:56:04)
# [GCC 8.4.0]
# Embedded file name: build/bdist.macosx-10.3-fat/egg/schevo/store/run_durus.py
# Compiled at: 2007-03-21 14:34:41
"""
$URL: svn+ssh://svn/repos/trunk/durus/run_durus.py $
$Id: run_durus.py 28260 2006-04-21 14:49:11Z dbinger $
"""
import sys, socket
from optparse import OptionParser
from schevo.store.storage_server import DEFAULT_PORT, DEFAULT_HOST, StorageServer
from schevo.store.storage_server import SocketAddress
from schevo.store.file_storage import FileStorage, TempFileStorage
from schevo.store.logger import log, logger, direct_output

def start_durus(logfile, logginglevel, file, repair, readonly, address):
    if logfile is None:
        logfile = sys.stderr
    else:
        logfile = open(logfile, 'a+')
    direct_output(logfile)
    logger.setLevel(logginglevel)
    if file is None:
        storage = TempFileStorage()
    else:
        storage = FileStorage(file, repair=repair, readonly=readonly)
    socket_address = SocketAddress.new(address)
    log(20, 'Storage file=%s address=%s', storage.fp.name, socket_address)
    StorageServer(storage, address=socket_address).serve()
    return

def stop_durus(address):
    sock = SocketAddress.new(address).get_connected_socket()
    if sock is None:
        raise SystemExit("Durus server %s doesn't seem to be running." % repr(address))
    sock.send('Q')
    sock.close()
    return

def run_durus_main():
    parser = OptionParser()
    parser.set_description('Run a Durus Server')
    parser.add_option('--port', dest='port', default=DEFAULT_PORT, type='int', help='Port to listen on. (default=%s)' % DEFAULT_PORT)
    parser.add_option('--file', dest='file', default=None, help='If this is not given, the storage is in a new temporary file.')
    parser.add_option('--host', dest='host', default=DEFAULT_HOST, help='Host to listen on. (default=%s)' % DEFAULT_HOST)
    if hasattr(socket, 'AF_UNIX'):
        parser.add_option('--address', dest='address', default=None, help='Address of the server.\nIf given, this is the path to a Unix domain socket for the server.')
        parser.add_option('--owner', dest='owner', default=None, help='Owner of the Unix domain socket (the --address value).')
        parser.add_option('--group', dest='group', default=None, help='group of the Unix domain socket (the --address value).')
        parser.add_option('--umask', dest='umask', default=None, type='int', help='umask for the Unix domain socket (the --address value).')
    logginglevel = logger.getEffectiveLevel()
    parser.add_option('--logginglevel', dest='logginglevel', default=logginglevel, type='int', help='Logging level. Lower positive numbers log more. (default=%s)' % logginglevel)
    parser.add_option('--logfile', dest='logfile', default=None, help='Log file. (default=stderr)')
    parser.add_option('--repair', dest='repair', action='store_true', help='Repair the filestorage by truncating to remove anything that is malformed.  Without this option, errors will cause the program to report and terminate without attempting any repair.')
    parser.add_option('--readonly', dest='readonly', action='store_true', help='Open the file in read-only mode.')
    parser.add_option('--stop', dest='stop', action='store_true', help='Instead of starting the server, try to stop a running one.')
    (options, args) = parser.parse_args()
    if getattr(options, 'address', None) is None:
        address = SocketAddress.new((options.host, options.port))
    else:
        address = SocketAddress.new(address=options.address, owner=options.owner, group=options.group, umask=options.umask)
    if not options.stop:
        start_durus(options.logfile, options.logginglevel, options.file, options.repair, options.readonly, address)
    else:
        stop_durus(address)
    return

if __name__ == '__main__':
    run_durus_main()


from django.shortcuts import redirect, render, HttpResponse
import random
c= random.randint(1, 100)

def init(request):
    request.session['counter']=0
    return redirect('')

def theprime(request):
    print(c)

    # count = request.session['counter']
    # count = count+1
    # request.session['counter']=count
    # context={
    #     'counterr':request.session['counter']
    # }

    return render(request,"prime.html")

# def con(request):
#     return redirect('post/index')

def index(request):

    x=int(request.POST['number'])
    print(c)
    context={'numberr':x

    }
    if x>c:

        return render(request,"high.html")
    elif (x<c):
        return render(request,"low.html")
    elif (x==c):
        return render(request,"equal.html",context)


#!/usr/bin/env python

import tornado.httpserver
import tornado.ioloop
import tornado.web
import tornado.auth
from tornado.web import HTTPError

from tornado.escape import json_encode as dumps
from tornado.escape import json_decode as loads

import db
import settings

# Major TODO: Handle exceptions for all SQL operations

class MainHandler(tornado.web.RequestHandler):
    """Main Handler... list all databases"""

    def get(self):
        self.write(dumps(db.list_databases()))

class ListTableHandler(tornado.web.RequestHandler):
    """List tables in specified database"""

    def get(self,database):
        try:
            self.write(dumps(db.list_tables(database)))
        except db.NoSuchDatabase:
            raise HTTPError(404)

class DataHandler(tornado.web.RequestHandler):
    def get(self,database,table,rowid=None):
        """Dumps a single record or all records from a table"""
        try:
            if rowid:
                self.write(dumps(db.get_record(database,table,rowid)))
            else:
                self.write(dumps([row
                    for row in db.all_records(database,table)]))
        except db.NoSuchDatabase:
            raise HTTPError(404)

    def post(self,database,table,rowid=None):
        """INSERT or UPDATE records"""
        # Prepare request (POST) vars. We only have to do this because
        # we get a dict of lists and our funcs want a dict of single
        # values
        kwargs = {}
        for k,v in self.request.arguments.iteritems():
            kwargs[k] = v[0]

        try:
            if rowid:
                # Perform UPDATE
                json = dumps(db.update_record(database,table,rowid,**kwargs))
                self.write(json)

            else:
                # Perform INSERT
                id = db.insert_record(database,table,**kwargs)
                self.set_status = 201   # HTTP 201 Created
                loc = '/%s/%s/%d' % (database,table,id)
                self.set_header('Location', loc)
                self.write(loc)

        except db.NoSuchDatabase:
            raise HTTPError(404)

    def put(self,database,table,rowid=None):
        """REPLACE record"""

        if not rowid:
            raise HTTPError(405) # We need a rowid to use REPLACE

        obj = loads(self.request.body)
        try:
            db.replace_record(database,table,rowid,obj)
        except db.NoSuchDatabase:
            raise HTTPError(404)

    def delete(self,database,table,rowid=None):
        """DELETE record"""

        if not rowid:
            raise HTTPError(405) # Need ROWID

        try:
            db.delete_record(database,table,rowid)
        except db.NoSuchDatabase:
            raise HTTPError(404)

application = tornado.web.Application([
    (r"/", MainHandler),
    (r"/([\w_\-\.]+)/", ListTableHandler),
    (r"/([\w_\-\.]+)/([\w]+)/([\d]+)?", DataHandler),
],
    cookie_secret=settings.cookie_secret,
)

if __name__ == "__main__":
    http_server = tornado.httpserver.HTTPServer(application)
    http_server.listen(settings.port)
    tornado.ioloop.IOLoop.instance().start()


__author__ = 'Titi'

import videodataset
from model.model_factory import Model
from utils.tools import init_meters
from utils.config import process_command

import numpy as np
import matplotlib.pyplot as plt
import argparse
import math
import random
import os
import cv2
from tqdm import tqdm

import torch
from torch.utils.data import DataLoader
from torch.utils.tensorboard import SummaryWriter

def main():
    args = process_command()
    device = torch.device('cuda:0' if torch.cuda.is_available() else 'cpu')

    # Loading LSTM model
    print("Loading LSTM model...", end='')
    LSTM = Model(args, device)
    LSTM.network.eval()
    print("...OK")

    # get video list from video_list_paths
    with open(args.valid_data_paths, 'r') as f:
        valid_video_list = [line.strip() for line in f.readlines()]

    # Loading model parameter
    if os.path.isfile(args.checkpoint_path):
        checkpoint = torch.load(args.checkpoint_path)
        model_state_dict = checkpoint['model_state_dict']
        optimizer_state_dict = checkpoint['optimizer_state_dict']
        pretrained_epoch = checkpoint['epoch']
        mask_probability = checkpoint['mask_probability']
        # model loading weight
        LSTM.load_checkpoint(model_state_dict, optimizer_state_dict)
    else:
        raise "[Error] No this checkpoint file"

    # Create validation dataset
    valid_dataset = videodataset.VideoDataset(valid_video_list, args.dataset_name,
                                              seq_length=args.seq_length,
                                              img_size=args.img_size,
                                              img_channel=args.img_channel,
                                              mode="valid",
                                              mask_probability=0)
    valid_loader = DataLoader(dataset=valid_dataset, batch_size=args.batch_size, shuffle=False, num_workers=args.num_workers)

    # validation
    print("Testing...")
    psnrs, ssims = [], []
    for _ in range(args.seq_length):
        init_psnr, init_ssim = init_meters()
        psnrs.append(init_psnr)
        ssims.append(init_ssim)

    with torch.no_grad():
        for vid_path, seq, seq_gt in tqdm(valid_loader):
            psnrs, ssims = LSTM.test(vid_path, args.gen_frm_dir, seq, seq_gt, 0, psnrs, ssims)

    for t in range(args.seq_length):
        print("{}\tPSNR: {:0.3f}, SSIM: {:0.3f}".format(t+1, psnrs[t].avg, ssims[t].avg))

if __name__ == '__main__':
    main()


import cv2
import operator

######################################################################
class ContourWithData ():
    # member variables
    npaContour = None  # contour
    boundingRect = None  # bounding rect for contour
    intRectX = 0  # bounding rect top left corner x location
    intRectY = 0  # bounding rect top left corner y location
    intRectWidth = 0  # bounding rect width
    intRectHeight = 0  # bounding rect height
    fltArea = 0.0  # area of contour
    aspectratio = 0  # aspect ratio of the contour formed
    avgwidth = 0
    avgheight =0

    def Calculate_Parameters(self):  # calculate bounding rect info
        [intX, intY, intWidth, intHeight] = self.boundingRect
        self.intRectX = intX
        self.intRectY = intY
        self.intRectWidth = intWidth
        self.intRectHeight = intHeight
        self.aspectratio = intWidth / intHeight;

def main(path,img):
    # Read the Image

    frame = cv2.imread(path + img)
    blurred_frame = cv2.GaussianBlur(frame, (5, 5), 0)
    hsv = cv2.cvtColor(blurred_frame, cv2.COLOR_BGR2HSV)

    lower = {'red': (166, 84, 141),'orange': (0, 50, 80)}  # assign new item lower['blue'] = (93, 10, 0)
    upper = {'red': (186, 255, 255),'orange': (20, 255, 255)}

    mask = None

    for key, value in upper.items():
        mask = cv2.inRange(hsv, lower[key], upper[key])

    allContoursWithData = []
    mask1=mask.copy()
    contours, _ = cv2.findContours(mask1, cv2.RETR_TREE, cv2.CHAIN_APPROX_NONE)

    ValidContour = []

    for i in contours:  # for each contour
        contourWithData = ContourWithData()  # instantiate a contour with data object
        contourWithData.npaContour = i  # assign contour to contour with data
        contourWithData.boundingRect = cv2.boundingRect ( contourWithData.npaContour )  # get the bounding rect
        contourWithData.Calculate_Parameters ()  # get bounding rect info
        contourWithData.fltArea = cv2.contourArea ( contourWithData.npaContour )  # calculate the contour area
        allContoursWithData.append ( contourWithData )  # add contour with data object to list of all contours with data

    for contour in allContoursWithData:
        if contour.fltArea > 0:
            ValidContour.append(contour)

    if len(ValidContour)==0:
        cv2.imwrite(path+"/Image_Processing/significant_img.jpg",frame)
        print("No Insignificant Numbers found.")
        return

    ValidContour.sort(key=operator.attrgetter("intRectX"))
    cv2.drawContours(frame, contours, -1, (54,255,255),5)
    h, w = frame.shape[:2]
    x=None

    if ValidContour[0].intRectX<(w/4):
        for i in range(len(ValidContour)):
            if ValidContour[i].intRectX>w/4:
                x=ValidContour[i].intRectX
    else:
        x=ValidContour[0].intRectX
    initial_y=ValidContour[0].intRectY

    y=initial_y+h

    #v = cv2.line(frame,(x,0),(x,y),(0,255,0),2)

    crop = frame[0:0 + y, 0:0 + x]

   #cv2.imshow ( "Significant_Digits Image", crop )
   # cv2.waitKey ( 0 )

    cv2.imwrite(path+"Image_Processing/significant_img.jpg",crop)
    print ( "Saving Significant Numbers." )
    return

if __name__ == '__main__':
    path= "C:/Users/Dushyant Sharma/Desktop/dataset/"
    main(path,"11.jpg")
    a=cv2.imread(path+"11.jpg")
    b=cv2.imread(path+"Image_Processing/significant_img.jpg")
    cv2.imshow("Original Image",a)
    cv2.imshow("Cropped Image",b)
    cv2.waitKey(0)


# Copyright 2013 Google Inc. All Rights Reserved.

"""A module to make it easy to set up and run CLIs in the Cloud SDK."""

import urllib
import urlparse
import uuid

from oauth2client import client

from googlecloudsdk.core import config
from googlecloudsdk.core import exceptions
from googlecloudsdk.core import log
from googlecloudsdk.core.credentials import store as c_store
from googlecloudsdk.core.util import console_io

__all__ = ['Credentials', 'Http']

class Error(exceptions.Error):
  """Exceptions for the cli module."""

class CannotRefreshAuthTokenError(Error, client.AccessTokenRefreshError):
  """An exception raised when the auth tokens fail to refresh."""

  def __init__(self, msg):
    auth_command = '$ gcloud auth login'
    message = ('There was a problem refreshing your current auth tokens: '
               '{0}.  Please run\n  {1}.'.format(msg, auth_command))
    super(CannotRefreshAuthTokenError, self).__init__(message)

def Credentials():
  """Get the currently active credentials.

  This function loads account credentials via core.account property
  of core.properties module.

  These credentials will be refreshed before being returned, so it makes sense
  to cache the value returned for short-lived programs.

  Returns:
    An active, valid credentials object.

  Raises:
    c_store.Error: If an error loading the credentials occurs.
  """
  return c_store.Load()

def Http(cmd_path=None, trace_token=None,
         auth=True, creds=None, timeout=None, log_http=False):
  """Get an httplib2.Http object for working with the Google API.

  Args:
    cmd_path: str, Path of command that will use the httplib2.Http object.
    trace_token: str, Token to be used to route service request traces.
    auth: bool, True if the http object returned should be authorized.
    creds: oauth2client.client.Credentials, If auth is True and creds is not
        None, use those credentials to authorize the httplib2.Http object.
    timeout: double, The timeout in seconds to pass to httplib2.  This is the
        socket level timeout.  If timeout is None, timeout is infinite.
    log_http: bool, Enable/disable client side logging of service requests.

  Returns:
    An authorized httplib2.Http object, or a regular httplib2.Http object if no
    credentials are available.
  """

  # TODO(user): Have retry-once-if-denied logic, to allow client tools to not
  # worry about refreshing credentials.

  http = c_store._Http(timeout=timeout)  # pylint:disable=protected-access

  # Wrap first to dump any data added by other wrappers.
  if log_http:
    http = _WrapRequestForLogging(http)

  # Wrap the request method to put in our own user-agent, and trace reporting.
  gcloud_ua = 'gcloud/{0} command/{1} invocation-id/{2} interactive/{3}'.format(
      config.CLOUD_SDK_VERSION,
      cmd_path,
      uuid.uuid4().hex,
      console_io.IsInteractive())
  http = _WrapRequestForUserAgentAndTracing(http, trace_token,
                                            gcloud_ua)
  if auth:
    if not creds:
      creds = Credentials()
    http = creds.authorize(http)
    # Wrap the request method to put in our own error handling.
    http = _WrapRequestForAuthErrHandling(http)
  return http

def _WrapRequestForUserAgentAndTracing(http, trace_token,
                                       gcloud_ua):
  """Wrap request with user-agent, and trace reporting.

  Args:
    http: The original http object.
    trace_token: str, Token to be used to route service request traces.
    gcloud_ua: str, User agent string to be included in the request.

  Returns:
    http, The same http object but with the request method wrapped.
  """
  orig_request = http.request

  def RequestWithUserAgentAndTracing(*args, **kwargs):
    """Wrap request with user-agent, and trace reporting.

    Args:
      *args: Positional arguments.
      **kwargs: Keyword arguments.

    Returns:
      Wrapped request method with user-agent and trace reporting.
    """
    modified_args = list(args)

    # Use gcloud specific user-agent with command path and invocation-id.
    # Pass in the user-agent through kwargs or args.
    def UserAgent(current=''):
      user_agent = '{0} {1}'.format(current, gcloud_ua)
      return user_agent.strip()
    if 'headers' in kwargs:
      cur_ua = kwargs['headers'].get('user-agent', '')
      kwargs['headers']['user-agent'] = UserAgent(cur_ua)
    elif len(args) > 3:
      cur_ua = modified_args[3].get('user-agent', '')
      modified_args[3]['user-agent'] = UserAgent(cur_ua)
    else:
      kwargs['headers'] = {'user-agent': UserAgent()}

    # Modify request url to enable requested tracing.
    url_parts = urlparse.urlsplit(args[0])
    query_params = urlparse.parse_qs(url_parts.query)
    if trace_token:
      query_params['trace'] = 'token:{0}'.format(trace_token)

    # Replace the request url in the args
    modified_url_parts = list(url_parts)
    modified_url_parts[3] = urllib.urlencode(query_params, doseq=True)
    modified_args[0] = urlparse.urlunsplit(modified_url_parts)

    return orig_request(*modified_args, **kwargs)

  http.request = RequestWithUserAgentAndTracing

  # apitools needs this attribute to do credential refreshes during batch API
  # requests.
  if hasattr(orig_request, 'credentials'):
    setattr(http.request, 'credentials', orig_request.credentials)

  return http

def _WrapRequestForAuthErrHandling(http):
  """Wrap request with exception handling for auth.

  We need to wrap exception handling because oauth2client does similar wrapping
  when you authorize the http object.  Because of this, a credential refresh
  error can get raised wherever someone makes an http request.  With no common
  place to handle this exception, we do more wrapping here so we can convert it
  to one of our typed exceptions.

  Args:
    http: The original http object.

  Returns:
    http, The same http object but with the request method wrapped.
  """
  orig_request = http.request

  def RequestWithErrHandling(*args, **kwargs):
    try:
      return orig_request(*args, **kwargs)
    except client.AccessTokenRefreshError as e:
      log.debug('Exception caught during HTTP request: %s', e.message,
                exc_info=True)
      raise CannotRefreshAuthTokenError(e.message)

  http.request = RequestWithErrHandling

  # apitools needs this attribute to do credential refreshes during batch API
  # requests.
  if hasattr(orig_request, 'credentials'):
    setattr(http.request, 'credentials', orig_request.credentials)

  return http

def _WrapRequestForLogging(http):
  """Wrap request for capturing and logging of http request/response data.

  Args:
    http: httplib2.Http, The original http object.

  Returns:
    http, The same http object but with the request method wrapped.
  """

  orig_request = http.request

  def RequestWithLogging(*args, **kwargs):
    """Wrap request for request/response logging.

    Args:
      *args: Positional arguments.
      **kwargs: Keyword arguments.

    Returns:
      Original returned response of this wrapped request.
    """
    _LogRequest(*args, **kwargs)
    response = orig_request(*args, **kwargs)
    _LogResponse(response)
    return response

  http.request = RequestWithLogging

  # apitools needs this attribute to do credential refreshes during batch API
  # requests.
  if hasattr(orig_request, 'credentials'):
    setattr(http.request, 'credentials', orig_request.credentials)

  return http

def _LogRequest(*args, **kwargs):
  """Logs request arguments."""

  log.status.Print('--request-start--')
  log.status.Print('uri: {uri}'.format(uri=args[0]))
  log.status.Print('-headers-start-')
  for h, v in sorted(kwargs.get('headers', {}).iteritems()):
    log.status.Print('{0}: {1}'.format(h, v))
  log.status.Print('-headers-end-')
  log.status.Print('-body-start-')
  if 'body' in kwargs:
    log.status.Print(kwargs['body'])
  log.status.Print('-body-end-')
  log.status.Print('--request-end--')

def _LogResponse(response):
  """Logs response headers and content."""

  headers, content = response
  log.status.Print('--response-start--')
  log.status.Print('-headers-start-')
  for h, v in sorted(headers.iteritems()):
    log.status.Print('{0}: {1}'.format(h, v))
  log.status.Print('-headers-end-')
  log.status.Print('-body-start-')
  log.status.Print(content)
  log.status.Print('-body-end-')
  log.status.Print('--response-end--')


import scrapy
class tpbSpider(scrapy.Spider):
	name = "TPB"

	start_urls = [
		'http://www.info.gov.hk/tpb/en/plan_application/application_comment_all.html'
	]

	def parse(self,response):
		for tr in response.css('tr'):
			yield{
			"application number" : tr.css("a[href^=java]::text").extract_first(),
			"date of meeting" : tr.css("td>div::text").re(r"^\d\d/\d\d/\d\d\d\d.*")
			}


class Solution:
    def countBits(self, num):
        """
        :type num: int
        :rtype: List[int]
        """
        powOfTwo = 1
        dp = [0, 1]
        for i in range(2, num+1):
            if powOfTwo*2 == i:
                dp.append(1)
                powOfTwo *= 2
            else:
                dp.append(1+dp[i-powOfTwo])
        return dp[:num+1]


import logging
import requests
from time import time

try:
  from google.appengine.ext import ndb
except e:
  logging.warn('appengine not found, NDBConfigSource will not work')

class NDBConfig(ndb.Model):
  props = ndb.JsonProperty()

class PropertySource(object):
  """A source of properties."""

  # time to live in seconds
  ttl = 60

  def fetch(self, existing_props={}):
    """Fetch properties.

    existing_props - all the properties that have been fetched so far
    return dictionary of properties
    """
    raise ValueError('should be implemented by subclass')

class NDBConfigSource(PropertySource):
  """Fetch/Store properties from appengine ndb"""

  def __init__(self, key):
    self.key = key

  def fetch(self, existing_props={}):
    """Fetch properties from ndb for a given key.

    return properties or empty {} if not found
    raise ValueError when not in appengine context
    """
    if not ndb:
      raise ValueError('not inside appengine context')

    config = NDBConfig.get_by_id(self.key)
    if not config:
      return {}

    return config.props

  def store(self, props):
    """Update ndb config entry"""
    config = NDBConfig(id = self.key, props = props).put()
    return props

class ConfigServiceSource(PropertySource):

  def __init__(self, namespace=None):
    """Define a config source from centralized config service.

    namespace - the config namespace
    """
    if not namespace:
      raise ValueError("url is required")

    self.namespace = namespace
    self.url = 'https://ragilconfig.appspot.com/config/%s' % self.namespace

  def fetch(self, existing_props={}):
    r = requests.get(self.url,
        auth=(existing_props['username'], existing_props['password']))

    if r.status_code == 200:
      return r.json()

    raise ValueError('Cannot fetch properties for namespace %s.\n%s' %
        (namespace, r.json()))

class Properties(object):

  def __init__(self, sources=[]):
    """Manages property lifecycle.

    sources - a list of PropertySource
    """
    self.sources = sources
    self.prop_cache = {}

  def fetch(self):
    """Return properties from all config sources"""
    props = {}
    for source in self.sources:

      prop = None

      # check if we still have valid props
      if source in self.prop_cache:
        if int(time()) - self.prop_cache[source]['ts_created'] < source.ttl:
          prop = self.prop_cache[source]['prop']

      if not prop:
        prop = source.fetch(props)
        self.prop_cache[source] = {
          'ts_created' : int(time()),
          'prop' : prop
        }

      props.update(prop)

    return props


#
# @lc app=leetcode.cn id=778 lang=python3
#
# [778] 水位上升的泳池中游泳
#
# https://leetcode-cn.com/problems/swim-in-rising-water/description/
#
# algorithms
# Hard (45.93%)
# Likes:    37
# Dislikes: 0
# Total Accepted:    2.6K
# Total Submissions: 5.7K
# Testcase Example:  '[[0,2],[1,3]]'
#
# 在一个 N x N 的坐标方格 grid 中，每一个方格的值 grid[i][j] 表示在位置 (i,j) 的平台高度。
#
# 现在开始下雨了。当时间为 t 时，此时雨水导致水池中任意位置的水位为 t
# 。你可以从一个平台游向四周相邻的任意一个平台，但是前提是此时水位必须同时淹没这两个平台。假定你可以瞬间移动无限距离，也就是默认在方格内部游动是不耗时的。当然，在你游泳的时候你必须待在坐标方格里面。
#
# 你从坐标方格的左上平台 (0，0) 出发。最少耗时多久你才能到达坐标方格的右下平台 (N-1, N-1)？
#
# 示例 1:
#
#
# 输入: [[0,2],[1,3]]
# 输出: 3
# 解释:
# 时间为0时，你位于坐标方格的位置为 (0, 0)。
# 此时你不能游向任意方向，因为四个相邻方向平台的高度都大于当前时间为 0 时的水位。
#
# 等时间到达 3 时，你才可以游向平台 (1, 1). 因为此时的水位是 3，坐标方格中的平台没有比水位 3 更高的，所以你可以游向坐标方格中的任意位置
#
#
# 示例2:
#
#
# 输入:
# [[0,1,2,3,4],[24,23,22,21,5],[12,13,14,15,16],[11,17,18,19,20],[10,9,8,7,6]]
# 输入: 16
# 解释:
# 0  1  2  3  4
# 24 23 22 21  5
# 12 13 14 15 16
# 11 17 18 19 20
# 10  9  8  7  6
#
# 最终的路线用加粗进行了标记。
# 我们必须等到时间为 16，此时才能保证平台 (0, 0) 和 (4, 4) 是连通的
#
#
# 提示:
#
#
# 2 <= N <= 50.
# grid[i][j] 位于区间 [0, ..., N*N - 1] 内。
#
#
#

# @lc code=start
class Solution:
    def swimInWater(self, grid: List[List[int]]) -> int:
        N = len(grid)
        visited, pq = {(0,0)}, [(grid[0][0],0,0)]
        ans = 0
        while pq:
            d,r,c = heapq.heappop(pq)
            ans = max(ans,d)
            if r == c == N-1:
                return ans
            for cr, cc in ((r-1,c),(r+1,c),(r,c-1),(r,c+1)):
                if 0 <= cr < N and 0 <= cc < N and (cr,cc) not in visited:
                    heapq.heappush(pq, (grid[cr][cc],cr,cc))
                    visited.add((cr,cc))

# @lc code=end

778.水位上升的泳池中游泳.py


#!/usr/bin/env python3

import tac
from cfg import infer, linearize
import cse
import copy
import cfg as tac_cfg
import re, random, os

# ------------------------------------------------------------------------------
# liveness

_arg1_use = re.compile(r'add|sub|mul|div|mod|neg|and|or|xor|not|shl|shr|copy|ret|jz|jnz|jl|jle')
_arg2_use = re.compile(r'add|sub|mul|div|mod|and|or|xor|shl|shr|param')
_dest_def = re.compile(r'add|sub|mul|div|mod|neg|and|or|xor|not|shl|shr|const|copy|phi|call')

def use_set(instr):
    s = set()
    if _arg1_use.fullmatch(instr.opcode): s.add(instr.arg1)
    if _arg2_use.fullmatch(instr.opcode): s.add(instr.arg2)
    if instr.opcode == 'phi': s.update(instr.arg1.values())
    return s

def rewrite_use_temps_nonphi(instr, fn):
    if _arg1_use.fullmatch(instr.opcode):
        instr.arg1 = fn(instr.arg1)
    if _arg2_use.fullmatch(instr.opcode):
        instr.arg2 = fn(instr.arg2)

def def_set(instr):
    s = set()
    if _dest_def.fullmatch(instr.opcode): s.add(instr.dest)
    return s

def rewrite_temps(instr, fn):
    if _arg1_use.fullmatch(instr.opcode):
        instr.arg1 = fn(instr.arg1)
    if _arg2_use.fullmatch(instr.opcode):
        instr.arg2 = fn(instr.arg2)
    if instr.opcode == 'phi':
        for l, t in instr.arg1.items():
            instr.arg1[l] = fn(t)
    if _dest_def.fullmatch(instr.opcode):
        instr.dest = fn(instr.dest)

# ------------------------------------------------------------------------------
# crude SSA gen

def tmp_root(tmp):
    try: return tmp[:tmp.rindex('.')]
    except ValueError: return tmp

def tmp_version(tmp):
    try: return tmp[tmp.rindex('.')+1:]
    except ValueError: return ''

def crude_ssagen(tlv, cfg):
    livein, liveout = dict(), dict()
    tac_cfg.recompute_liveness(cfg, livein, liveout)
    for bl in cfg.nodes():
        prev_labs = list(cfg.predecessors(bl.label))
        ts = livein[bl.first_instr()]
        if len(prev_labs) == 0: prev_labs = [cfg.proc_name]
        bl.body[:0] = [tac.Instr(t, 'phi', {l: t for l in prev_labs}, None) \
                       for t in ts]
    versions = tac_cfg.counter(transfn=lambda x: f'.{x}')
    for i in cfg.instrs():
        if i.dest: i.dest = i.dest + next(versions)
    ver_maps = {cfg.proc_name: {t: t for t in tlv.t_args}}
    for bl in cfg.nodes():
        ver_map = dict()
        for instr in bl.instrs():
            rewrite_use_temps_nonphi(instr, lambda t: ver_map.get(t, t))
            if instr.dest:
                ver_map[tmp_root(instr.dest)] = instr.dest
        ver_maps[bl.label] = ver_map
    for bl in cfg.nodes():
        for instr in bl.instrs():
            if instr.opcode != 'phi': continue
            for lab_prev, root in instr.arg1.items():
                instr.arg1[lab_prev] = ver_maps[lab_prev].get(root, root)

class ufset:
    def __init__(self):
        self._parents = dict()

    def __call__(self, x):
        if x not in self._parents: return x
        p = self(self._parents[x])
        self._parents[x] = p    # path compression
        return p

    def union(self, x, y):
        px = self(x)
        py = self(y)
        self._parents[px] = py

def ssa_minimize(tlv, cfg):
    teq = ufset()
    dirty = True
    change = False
    while dirty:
        dirty = False
        for instr in cfg.instrs():
            if instr.opcode != 'phi': continue
            lhs = teq(instr.dest)
            rhs = set(teq(t) for t in instr.arg1.values())
            # null choice
            if len(rhs) == 1 and lhs in rhs:
                dirty = True
                change = True
                instr.dest, instr.arg1 = None, None
                instr.opcode = 'nop'
            # rename
            if len(rhs.union({lhs})) == 2:
                dirty = True
                change = True
                if lhs in rhs: rhs.remove(lhs)
                teq.union(lhs, rhs.pop())
    tlv.t_args = tuple(teq(t) for t in tlv.t_args)
    for bl in cfg.nodes():
        bl.body = list(filter(lambda instr: instr.opcode != 'nop', bl.body))
    for instr in cfg.instrs(): rewrite_temps(instr, teq)
    return change

# ------------------------------------------------------------------------------

class IG:
    def __init__(self):
        self._nxt = dict()

    def add_edge(self, f, t):
        if f == t: return
        self._nxt.setdefault(f, set()).add(t)
        self._nxt.setdefault(t, set()).add(f)

    def remove_edge(self, f, t):
        if f == t: return
        self._nxt[f].discard(t)
        self._nxt[t].discard(f)

    def remove_node(self, n):
        for t in self._nxt[n]: self._nxt[t].discard(n)
        del self._nxt[n]

    def mcs(self):
        waiting = set(self._nxt.keys())
        wt = {n: 0 for n in waiting}
        order = []
        while len(waiting) > 0:
            v = max(waiting, key=lambda n: wt[n])
            order.append(v)
            for w in self._nxt[v]:
                if w not in waiting: continue
                wt[w] += 1
            waiting.remove(v)
        return order

    def color(self, *, pre=None, order=None):
        col = pre.copy() if pre else dict()
        order = order or self.mcs()
        for n in self._nxt: col.setdefault(n, 0)
        for v in order:
            if col[v] != 0: continue
            ncols = {col[w] for w in self._nxt[v]}
            k = 1
            while True:
                if k not in ncols: break
                k += 1
            col[v] = k
        for u, vs in self._nxt.items():
            for v in vs:
                assert col[u] != col[v]
        return col

    def write_dot(self, name, filename, col=None):
        with open(filename, 'w') as f:
            print(f'graph {name} {{\ngraph[overlap=false];', file=f)
            ids = {u: k for k, u in enumerate(self._nxt)}
            for u in self._nxt:
                k = col[u] if col else '?'
                print(f'{ids[u]} [label="{u}/{k}",fontname=monospace,fontsize=8];', file=f)
            for u, vs in self._nxt.items():
                for v in vs:
                    if v < u: continue
                    print(f'{ids[u]} -- {ids[v]};', file=f)
            print('}', file=f)
        os.system(f'neato -Tpdf -O {filename}')

# ------------------------------------------------------------------------------
def live_in(cfg):
    livein = dict()
    for instr in cfg.instrs():
        livein[instr] = use_set(instr)
    # run the livein set update loop until there are no more changes
    dirty = True
    while dirty:
        dirty = False
        for (i1, i2) in cfg.instr_pairs():
            t_live = livein[i2].difference(def_set(i1))
            if not t_live.issubset(livein[i1]):
                dirty = True # there was a change, so run it again
                livein[i1] = livein[i1].union(t_live)
    return livein

def live_out(cfg,livein):
        liveout = dict()
        for block in cfg._blockmap.values():
            for i,instr in enumerate(block.instrs()):
                block_instrs =list(block.instrs())
                for j in range(i+1,len(block_instrs)):
                    if instr in liveout.keys(): liveout[instr] = (livein[block_instrs[j]]).union(liveout[instr])
                    else: liveout[instr] = livein[block_instrs[j]]
                for succ_block in [cfg[label] for label in cfg.successors(block.label)]:
                    for succ_instr in succ_block.instrs():
                        if instr in liveout.keys(): liveout[instr] = (livein[succ_instr]).union(liveout[instr])
                        else: liveout[instr] = livein[succ_instr]
        return liveout

def dead_copy_elim(cfg):
    change = 0
    for b in cfg._blockmap.values():
        #print("BEFORE\n",b)
        to_delete = []
        for instr in b.body:
            # dead store
            livein = live_in(cfg)
            liveout = live_out(cfg,livein)

            if not def_set(instr).issubset(liveout[instr]):
                # call instr
                if instr.opcode == 'call':
                    if instr.dest != '%_':
                        change = 1
                        instr.dest = '%_'
                # arithmetic operation
                elif instr.opcode in ['add', 'sub', 'mul', 'div', 'mod', 'neg', 'and', 'or',
                                        'xor', 'not', 'shl', 'shr', 'const', 'copy']:
                    to_delete.append(instr)
                    change = 1

        for instr in reversed(to_delete):
            b.body.remove(instr)

        #print("AFTER\n",b)

    return change

def get_temps(cfg):
    #Function that returns the initial Val and Ev mappings
    Val,Ev = {},{}
    for block in cfg._blockmap.values():
        if block.label not in Ev:
            Ev[block.label]=False
        for instr in block.body:
            for v in [instr.dest,instr.arg1,instr.arg2]:
                if (type(v)==str and v[0]=="%") and v not in Val:
                    Val[v]="Bot"
    return Val,Ev

def sccp(cfg):
    #We fill Ev and Var with the initial block and arguments

    c_jumps = ['jz', 'jnz', 'jl', 'jle']
    Val,Ev = get_temps(cfg)
    #For every temp in input, we set Val(v) to top
    for temp in cfg.t_args:
        Val[temp]="Top"

    #For the initial block, we set Ev(B)=True
    Ev[list(cfg._blockmap.keys())[0]]=True

    #Visiting blocks and updating Ev
    order = list(cfg._blockmap.keys())
    random.shuffle(order)

    #We then repeat the below process until there is no change in Ev or Var
    changed = True
    while changed:

        changed = False

        for b in order:
            definite = False #Presence of a definite jump ()
            if Ev[b]==True:
                for instr in cfg._blockmap[b].body:
                    #Case of conditional jumps
                    if instr.opcode in c_jumps:
                        jmp_dst = instr.arg2 #Check if it is indeed arg2
                        used = use_set(instr)
                        if len(used)==1:
                            if Val[used]=="Top":
                                Ev[jmp_dst]=True
                                changed = True
                            elif Val[used]=="Bot":
                            #stop further updates based on this and later jumps in B
                                break
                            else:
                                x = Val[used]
                                if instr.opcode=="jz":
                                    if x==0:
                                        Ev[jmp_dst],definite=True,True
                                        changed = True
                                if instr.opcode == "jnz":
                                    if x!=0:
                                        Ev[jmp_dst],definite=True,True
                                        changed = True
                                if instr.opcode == "jl":
                                    if x<0:
                                        Ev[jmp_dst],definite=True,True
                                        changed = True
                                if instr.opcode == "jle":
                                    if x<=0:
                                        Ev[jmp_dst],definite=True,True
                                        changed = True

                    #Case of jump
                    elif instr.opcode=="jmp":
                        if definite==False:
                            Ev[instr.arg1]=True
                            changed = True

                    else:
                        used = use_set(instr)
                        #Cases of phi instruction (numbered as in the pdf of the project task)
                        if instr.opcode=="phi":
                            used_temps = list(instr.arg1.values())
                            block_labels = list(instr.arg1.keys())

                            vals = [Val[i] for i in used_temps]
                            setv = set(vals)

                            #Case 5
                            if len(setv)==1 and list(setv)[0] not in ["Top","Bot"]:
                                Val[instr.dest] = val
                                changed = True

                            #Case 3
                            elif len(setv)==2 and "Bot" in setv:
                                cop = list(setv)
                                cop.remove("Bot")
                                if cop[0] not in ["Top","Bot"]:
                                    Val[instr.dest] = val
                                    changed = True
                                    break

                            else:
                                #Keep constant value
                                cst = []

                                for temp in used_temps:
                                    val = Val[temp]

                                    if val not in ["Top","Bot"]:

                                        cst.append(temp)

                                        #Case 1
                                        if len(set(cst))==2:
                                            Val[instr.dest] = "Top"
                                            changed = True
                                            break

                                    #Case 2
                                    if val=="Top":
                                        #We go fetch the block from where it comes from
                                        origin_block = block_labels[used_temps.index(temp)]
                                        if Ev[origin_block]==True:
                                            Val[instr.dest] = "Top"
                                            changed = True
                                            break

                                #Case 4
                                for c_temp in cst:
                                    others = used_temps.remove(c_temp)
                                    n = len(others)
                                    k = 0
                                    for oth in others:
                                        origin_block = block_labels[used_temps.index(oth)]
                                        if Ev[origin_block]==True: break
                                        else: k+=1

                                    if k==n:
                                        Val[instr.dest] = Val[c_temp]
                                        changed = True

                        else:
                            if len(used)==2:
                                x = used.pop()
                                y = used.pop()
                                if Val[x]=="Top" or Val[y]=="Top":
                                    for temp in self.def_set(instr):
                                        Val[temp]="Top"
                                        changed = True
                                elif (Val[x] not in ["Top","Bot"]) and (Val[y] not in ["Top","Bot"]):
                                    changed = True
                                    dst = self.def_set(instr)
                                    if instr.opcode=="add": Val[dst]= Val[x]+Val[y]
                                    elif instr.opcode=="sub": Val[dst]= Val[x]-Val[y]
                                    elif instr.opcode=="mul": Val[dst]= Val[x]*Val[y]
                                    elif instr.opcode=="div": Val[dst]= Val[x]/Val[y]
                                    elif instr.opcode=="mod": Val[dst]= Val[x]%Val[y]
                                    elif instr.opcode=="shl": Val[dst]= Val[x]<<Val[y]
                                    elif instr.opcode=="shr": Val[dst]= Val[x]>>Val[y]
                                    elif instr.opcode=="and": Val[dst]= Val[x]&Val[y]
                                    elif instr.opcode=="or": Val[dst]= Val[x]|Val[y]
                                    elif instr.opcode=="xor": Val[dst]= Val[x]^Val[y]

        #print(Ev,Val)

        #If the above code was working, we would then delete some instructions
        # according to Ev and Val.
# ------------------------------------------------------------------------------

def ssagen_and_optimise(loaded_tac):
    for tlv in loaded_tac:
        if isinstance(tlv, tac.Proc):
            cfg = tac_cfg.infer(tlv)
            crude_ssagen(tlv, cfg)
            dirty = True
            while dirty:
                dirty = False
                if ssa_minimize(tlv, cfg): dirty = True # rename and nce
                if cse.run_cse(cfg): dirty = True       # common sub-expression elimination
                if cfg.copy_propagate(): dirty = True   # copy propagation
                if dead_copy_elim(cfg)==1: dirty = True
                #sccp(cfg)     <- The function has problems
            livein, liveout = dict(), dict()
            tac_cfg.recompute_liveness(cfg, livein, liveout)
            tac_cfg.linearize(tlv, cfg)
    return loaded_tac

# ------------------------------------------------------------------------------

if __name__ == '__main__':
    import os
    from argparse import ArgumentParser
    ap = ArgumentParser(description='TAC library, parser, and interpreter')
    ap.add_argument('file', metavar='FILE', type=str, nargs=1, help='A TAC file')
    ap.add_argument('-v', dest='verbosity', default=0, action='count',
                    help='increase verbosity')
    args = ap.parse_args()
    gvars, procs = dict(), dict()
    for tlv in tac.load_tac(args.file[0]):
        if isinstance(tlv, tac.Proc):
            cfg = tac_cfg.infer(tlv)
            crude_ssagen(tlv, cfg)
            ssa_minimize(tlv, cfg)
            livein, liveout = dict(), dict()
            tac_cfg.recompute_liveness(cfg, livein, liveout)
            tac_cfg.linearize(tlv, cfg)
            if args.verbosity >= 2:
                print(tlv)


from phue import Bridge
import time

# Bridge IP
b = Bridge('192.168.1.3')

b.connect()

lights = b.get_light_objects('name')

# Set the lights
for light in ['Living Room 1', 'Living Room 2']:
  lights[light].on = True
  lights[light].brightness = 255

i = 0
maxIterations = 5
while i < maxIterations:
  for light in ['Living Room 1', 'Living Room 2']:
  	# Top color temp is 6500k
    lights[light].colortemp_k = 6500
  time.sleep(1)
  for light in ['Living Room 1', 'Living Room 2']:
  	# Bottom color temp is 2000k
    lights[light].colortemp_k = 2000
  time.sleep(1)
  # Increment i so we eventually break the loop
  i += 1

# Tear down the lights
for light in ['Living Room 1', 'Living Room 2']:
  lights[light].on = False


import datetime
import itertools
import time
import types

from selenium import webdriver

import vtk

def now():
    return datetime.datetime.now().strftime("%Y-%m-%d-%H:%M:%S")

def compare_images(test_img, baseline_img):
    # Create a vtkTesting object, and tell it to use the current directory as
    # the "baseline root directory", and specify the name of the baseline image
    # file itself.
    t = vtk.vtkTesting()
    t.AddArgument("-B")
    t.AddArgument(".")
    t.AddArgument("-V")
    t.AddArgument(baseline_img)

    # Perform the image comparison test and print out the result.
    t.RegressionTest(test_img, 0.0)

def wait_with_timeout(delay=None, limit=None, criterion=None):
    for i in itertools.count():
        if criterion():
            return True
        elif delay * i > limit:
            return False
        else:
            time.sleep(delay)

browsers = ["firefox", "chrome", "internet_explorer"]
Browsers = type("Enum", (), {k: i for i, k in enumerate(browsers)})

class WebTest(object):
    class Abort:
        pass

    def __init__(self, size=None, url=None, browser=None):
        self.size = size
        self.url = url
        self.browser = browser

        self.window = None

    def run_test(self):
        result = None
        try:
            self.initialize()
            self.setup()
            self.capture()
            result = self.postprocess()
        except WebTest.Abort:
            result = 1

        self.cleanup()
        return result

    def initialize(self):
        if self.browser is None or self.browser == Browsers.chrome:
            self.window = webdriver.Chrome()
        elif self.browser == Browsers.firefox:
            self.window = webdriver.Firefox()
        elif self.browser == Browsers.internet_explorer:
            self.window = webdriver.Ie()
        else:
            raise ValueError("self.browser argument has illegal value %r" %
                             (self.browser))

        if self.size is not None:
            self.window.set_window_size(self.size[0], self.size[1])

        if self.url is not None:
            self.window.get(self.url)

    def setup(self):
        pass

    def capture(self):
        self.filename = "%s-%s.png" % (self.__class__name, now())
        self.window.save_screenshot(filename)

    def postprocess(self):
        pass

    def cleanup(self):
        self.window.quit()

class CDashImageComparator(WebTest):
    def __init__(self, filepat=None, baseline=None, **kwargs):
        if filepat is None:
            raise TypeError("missing argument 'filepat'")
        if baseline is None:
            raise TypeError("missing argument 'baseline'")

        WebTest.__init__(self, **kwargs)
        self.filepat = filepat
        self.baseline = baseline

    def capture(self):
        self.filename = self.filepat % (now())
        self.window.save_screenshot(self.filename)

    def postprocess(self):
        compare_images(self.filename, self.baseline)


import os
import logging
from importlib import import_module

from django.conf import settings
from django.http import HttpResponse, HttpResponseServerError
from django.views.decorators.http import require_POST
from django.views.decorators.csrf import csrf_exempt

from djangae.utils import on_production
from djangae.core.signals import module_started, module_stopped

def warmup(request):
    """
        Provides default procedure for handling warmup requests on App
        Engine. Just add this view to your main urls.py.
    """
    for app in settings.INSTALLED_APPS:
        for name in ('urls', 'views', 'models'):
            try:
                import_module('%s.%s' % (app, name))
            except ImportError:
                pass
    return HttpResponse("Ok.")

def start(request):
    module_started.send(sender=__name__, request=request)
    return HttpResponse("Ok.")

def stop(request):
    module_stopped.send(sender=__name__, request=request)
    return HttpResponse("Ok.")

@csrf_exempt
def deferred(request):
    from google.appengine.ext.deferred.deferred import (
        run,
        SingularTaskFailure,
        PermanentTaskFailure
    )

    response = HttpResponse()

    if 'HTTP_X_APPENGINE_TASKEXECUTIONCOUNT' in request.META:
        logging.debug("[DEFERRED] Retry %s of deferred task", request.META['HTTP_X_APPENGINE_TASKEXECUTIONCOUNT'])

    if 'HTTP_X_APPENGINE_TASKNAME' not in request.META:
        logging.critical('Detected an attempted XSRF attack. The header "X-AppEngine-Taskname" was not set.')
        response.status_code = 403
        return response

    in_prod = on_production()

    if in_prod and os.environ.get("REMOTE_ADDR") != "0.1.0.2":
        logging.critical('Detected an attempted XSRF attack. This request did not originate from Task Queue.')
        response.status_code = 403
        return response

    try:
        run(request.body)
    except SingularTaskFailure:
        logging.debug("Failure executing task, task retry forced")
        response.status_code = 408
    except PermanentTaskFailure:
        logging.exception("Permanent failure attempting to execute task")

    return response

@csrf_exempt
@require_POST
def internalupload(request):
    try:
        return HttpResponse(str(request.FILES['file'].blobstore_info.key()))
    except Exception:
        logging.exception("DJANGAE UPLOAD FAILED: The internal upload handler couldn't retrieve the blob info key.")
        return HttpResponseServerError()


#
# Ruler.py -- Ruler plugin for Ginga reference viewer
#
# This is open-source software licensed under a BSD license.
# Please see the file LICENSE.txt for details.
#
from ginga import GingaPlugin
from ginga.gw import Widgets

class Ruler(GingaPlugin.LocalPlugin):

    def __init__(self, fv, fitsimage):
        # superclass defines some variables for us, like logger
        super(Ruler, self).__init__(fv, fitsimage)

        self.rulecolor = 'green'
        self.layertag = 'ruler-canvas'
        self.ruletag = None

        self.dc = fv.getDrawClasses()
        canvas = self.dc.DrawingCanvas()
        canvas.enable_draw(True)
        canvas.enable_edit(True)
        canvas.set_drawtype('ruler', color='cyan')
        canvas.set_callback('draw-event', self.wcsruler)
        canvas.set_callback('draw-down', self.clear)
        canvas.set_callback('edit-event', self.edit_cb)
        canvas.set_draw_mode('draw')
        canvas.set_surface(self.fitsimage)
        canvas.register_for_cursor_drawing(self.fitsimage)
        canvas.name = 'Ruler-canvas'
        self.canvas = canvas

        self.w = None
        self.unittypes = ('arcmin', 'degrees', 'pixels')
        self.units = 'arcmin'

    def build_gui(self, container):
        top = Widgets.VBox()
        top.set_border_width(4)

        vbox, sw, orientation = Widgets.get_oriented_box(container)
        vbox.set_border_width(4)
        vbox.set_spacing(2)

        self.msgFont = self.fv.getFont("sansFont", 12)
        tw = Widgets.TextArea(wrap=True, editable=False)
        tw.set_font(self.msgFont)
        self.tw = tw

        fr = Widgets.Expander("Instructions")
        fr.set_widget(tw)
        vbox.add_widget(fr, stretch=0)

        fr = Widgets.Frame("Ruler")

        captions = (('Units:', 'label', 'Units', 'combobox'),
                    )
        w, b = Widgets.build_info(captions, orientation=orientation)
        self.w = b

        combobox = b.units
        for name in self.unittypes:
            combobox.append_text(name)
        index = self.unittypes.index(self.units)
        combobox.set_index(index)
        combobox.add_callback('activated', lambda w, idx: self.set_units())

        fr.set_widget(w)
        vbox.add_widget(fr, stretch=0)

        mode = self.canvas.get_draw_mode()
        hbox = Widgets.HBox()
        btn1 = Widgets.RadioButton("Draw")
        btn1.set_state(mode == 'draw')
        btn1.add_callback('activated', lambda w, val: self.set_mode_cb('draw', val))
        btn1.set_tooltip("Choose this to draw a ruler")
        self.w.btn_draw = btn1
        hbox.add_widget(btn1)

        btn2 = Widgets.RadioButton("Edit", group=btn1)
        btn2.set_state(mode == 'edit')
        btn2.add_callback('activated', lambda w, val: self.set_mode_cb('edit', val))
        btn2.set_tooltip("Choose this to edit a ruler")
        self.w.btn_edit = btn2
        hbox.add_widget(btn2)

        hbox.add_widget(Widgets.Label(''), stretch=1)
        vbox.add_widget(hbox, stretch=0)

        spacer = Widgets.Label('')
        vbox.add_widget(spacer, stretch=1)

        top.add_widget(sw, stretch=1)

        btns = Widgets.HBox()
        btns.set_spacing(3)

        btn = Widgets.Button("Close")
        btn.add_callback('activated', lambda w: self.close())
        btns.add_widget(btn, stretch=0)
        btns.add_widget(Widgets.Label(''), stretch=1)
        top.add_widget(btns, stretch=0)

        container.add_widget(top, stretch=1)

    def set_units(self):
        index = self.w.units.get_index()
        units = self.unittypes[index]
        self.units = units
        self.canvas.set_drawtype('ruler', color='cyan', units=units)

        if self.ruletag is not None:
            obj = self.canvas.get_object_by_tag(self.ruletag)
            if obj.kind == 'ruler':
                obj.units = units
                self.canvas.redraw(whence=3)
        return True

    def close(self):
        self.fv.stop_local_plugin(self.chname, str(self))
        return True

    def instructions(self):
        self.tw.set_text("""Draw (or redraw) a line with the cursor.

Display the Zoom tab at the same time to precisely see detail while drawing.""")

    def start(self):
        self.instructions()
        # start ruler drawing operation
        p_canvas = self.fitsimage.get_canvas()
        if not p_canvas.has_object(self.canvas):
            p_canvas.add(self.canvas, tag=self.layertag)

        self.canvas.delete_all_objects()
        self.resume()

    def pause(self):
        self.canvas.ui_setActive(False)

    def resume(self):
        self.canvas.ui_setActive(True)
        self.fv.showStatus("Draw a ruler with the right mouse button")

    def stop(self):
        # remove the canvas from the image
        p_canvas = self.fitsimage.get_canvas()
        try:
            p_canvas.delete_object_by_tag(self.layertag)
        except:
            pass
        self.canvas.ui_setActive(False)
        self.fv.showStatus("")

    def redo(self):
        obj = self.canvas.get_object_by_tag(self.ruletag)
        if obj.kind != 'ruler':
            return True
        # redraw updates ruler measurements
        self.canvas.redraw(whence=3)

    def clear(self, canvas, button, data_x, data_y):
        self.canvas.delete_all_objects()
        self.ruletag = None
        return False

    def wcsruler(self, surface, tag):
        obj = self.canvas.get_object_by_tag(tag)
        if obj.kind != 'ruler':
            return True
        # remove the old ruler
        try:
            self.canvas.delete_object_by_tag(self.ruletag)
        except:
            pass

        # change some characteristics of the drawn image and
        # save as the new ruler
        self.ruletag = tag
        obj.color = self.rulecolor
        obj.cap = 'ball'
        self.canvas.redraw(whence=3)

    def edit_cb(self, canvas, obj):
        self.redo()
        return True

    def edit_select_ruler(self):
        if self.ruletag is not None:
            obj = self.canvas.get_object_by_tag(self.ruletag)
            self.canvas.edit_select(obj)
        else:
            self.canvas.clear_selected()
        self.canvas.update_canvas()

    def set_mode_cb(self, mode, tf):
        if tf:
            self.canvas.set_draw_mode(mode)
            if mode == 'edit':
                self.edit_select_ruler()
        return True

    def __str__(self):
        return 'ruler'

#END


#!/usr/bin/env python3
"""lab19_1.py A SortedDictionary class with only "description"
allowed as an attribute -- using __setattr__"""

class SortedDictionary(dict):

    allowed_attributes = "description",

    def keys(self):
        return sorted(dict.keys(self))
        # return sorted(super(type(self), self).keys())

    def __iter__(self):
        """If we don't define this, it will use the regular
        dictionary __iter__ which does not call
        SortedDictionary.keys()."""

        for each in list(self.keys()):
            yield each

    def __setattr__(self, attribute_name, value):
        if attribute_name not in SortedDictionary.allowed_attributes:
            raise TypeError(f"Can't set attributes of class {self.__class__.__name__}")
        self.__dict__[attribute_name] =  value

def main():
    # d is a tuple of contructs that will initialize
    # a regular dictionary.
    for initializer in (
            {"Zero":0, "False":0, "None":0, "True":1},
            {},
            (("calling birds", 4), ("french hens", 3),
            ("turtle doves", 2),
            ("partridge in a pear tree", 1))):
        regular_dict = dict(initializer)
        sorted_dict = SortedDictionary(initializer)
        print(f" regular_dict.keys(): {list(regular_dict.keys())}")
        print(f"  sorted_dict.keys(): {list(sorted_dict.keys())}")
        print(f"""sorted_dict for-loop: {", ".join([
            str(k) for k in sorted_dict])}""")

    sorted_dict.description = "Fourth Day of Christmas"
    print(f"sorted_dict.description = {sorted_dict.description}")
    try:
        regular_dict.description = "Fourth Day of Christmas"
    except AttributeError:
        pass
    else:
        print("Unexpected behavior!")
    sorted_dict.x = 3

if __name__ == "__main__":
    main()


import pickle as pickle
import sklearn
from sklearn.model_selection import train_test_split
import numpy as np
from numpy import argmax
import pylab as pl
from tqdm import tqdm
import json
#file = '/SNAPPcapAnalysis/NNdataMlabs.pickle'
# file = 'NNdataMlabs.pickle'
# with open(file, 'rb') as f:
#     load = pickle.load(f)
#
# print("loaded")
#file = '/home/vicente/PycharmProjects/pythonProject/RTTDATA/RTTOUT_1626295365.jsonl'

x_raw = []
y_raw= []

#print(load[0][0])
#print(load[0][1])
bins = []
for i in range(3):
    bins.append([])
count = 0
total_loss = 0
total = 0

x_loss = []
y_loss =[]

x_no_loss = []
y_no_loss = []
#IN_FILE = "NNdataMlabs.jsonl"
#IN_FILE = '/home/vicente/storage/NNdata50ms_NewCleanHopefully.jsonl'
IN_FILE = '/home/vicente/storage/JsonlMaster.jsonl'

# fpath = os.path.join(IN_DIR, IN_FILE)
import jsonlines
# for line in open(IN_FILE).readlines():
#     if line != '\n':
with jsonlines.open(IN_FILE) as f:
    for line in f.iter():
        if line != '\n':
            #window = json.loads(line)
            window = line

            #print(parsed_line)

    #for window in tqdm(load):
            trace = window[1]
            if count % 10000 == 0:
                print(count)
            window = window[0]
            count +=1
            #print(window)
            #temp = np.array(packet[0])
            #print(temp.dtype)
            #print(temp)
            if np.isnan(np.sum(np.array(window[0]))) or np.isnan(np.sum(np.array(window[1]))):
                continue
            #if count > 6000000:
            #if count > 6000000:
            #    break
            #x_raw.append(window[0])
            lossRate = 0.0
            if window[1] == 0 and count > 6000000:# and count < 6000000:
            #if window[1] == 0 and count < 3000000:
                #y_raw.append(0)
                x_no_loss.append(window[0])
                y_no_loss.append(0)
                #total = total +1
            #window = None
    #y_raw.append(window[1])
            # if count == 3000000:
            #     x_no_loss = np.asarray(x_no_loss)
            #     y_no_loss = np.asarray(y_no_loss)
            #     print(type(x_no_loss))
            #     print(type(y_no_loss))
            #
            #     num_classes = 4
            #     # y_loss = tf.keras.utils.to_categorical(y_loss, num_classes)
            #
            #     print("starting pickle")
            #     x_no_loss = np.array_split(x_no_loss, 5)
            #     y_no_loss = np.array_split(y_no_loss, 5)
            #     print(type(x_no_loss))
            #     print(type(y_no_loss))
            #     print(len(x_no_loss))
            #     print(len(y_no_loss))
            #     for i in range(5):
            #         print (y_no_loss[i])
            #     # size= []
            #     # offset = 15
            #     for j in range(5):
            #         print("Iteration: X", j)
            #         with open("/home/vicente/storage/data/x_no_loss" + str(j) + "50msListFilter.pickle", 'wb') as f:
            #             pickle.dump(x_no_loss[j], f)
            #         print(x_no_loss[j].shape)
            #         x_no_loss[j] = 0
            #
            #     for k in range(5):
            #         print(y_no_loss[k])
            #         print("Starting y asarray")
            #         print("Iteration: Y", k)
            #         with open("/home/vicente/storage/data/y_no_loss_data" + str(k) + "50msListFilter.pickle", 'wb') as f:
            #             pickle.dump(y_no_loss[k], f)
            #         # size.append(x_no_loss.shape[0])
            #         y_no_loss[i] = 0
            #         x_no_loss = []
            #         y_no_loss = []

x_no_loss = np.asarray(x_no_loss)
y_no_loss = np.asarray(y_no_loss)

num_classes = 4
#y_loss = tf.keras.utils.to_categorical(y_loss, num_classes)

print("starting pickle")
x_no_loss = np.array_split(x_no_loss,5)
y_no_loss = np.array_split(y_no_loss,5)

#size= []
offset = 10
for i in range(5):
    j=i + offset
    print("Iteration: X", j)
    with open("/home/vicente/storage/data/x_no_loss" + str(j) + "50msListFilter.pickle", 'wb') as f:
        pickle.dump(x_no_loss[i], f)
    print(x_no_loss[i].shape)
    x_no_loss[i] = 0

for i in range(5):
    j=i + offset
    print("Starting y asarray")
    print("Iteration: Y", j)
    with open("/home/vicente/storage/data/y_no_loss_data" + str(j) + "50msListFilter.pickle", 'wb') as f:
        pickle.dump(y_no_loss[i], f)
    #size.append(x_no_loss.shape[0])
    y_no_loss[i] = 0

# for i in range(5):
#     j = i + offset
#     print("Iteration: X", j)
#     with open("/home/vicente/storage/data/x_no_loss" + str(j) + "50ms.pickle", 'wb') as f:
#         pickle.dump(x_no_loss[i], f)
#     print(x_no_loss[i].shape)
#     x_no_loss[i] = 0

# for i in range(5):
#     j = i + offset
#     print("Starting y asarray")
#     print("Iteration: Y", j)
#     with open("/home/vicente/storage/data/y_no_loss_data" + str(j) + "50ms.pickle", 'wb') as f:
#         pickle.dump(y_no_loss[i], f)
#     #size.append(x_no_loss.shape[0])
#     y_no_loss[i] = 0
#

#print(size)
    # with open("y_no_loss_data" + str(i) + ".pickle", 'wb') as f:
    #     pickle.dump(y_no_loss[i], f)
    # y_no_loss[i] = None

#y_no_loss = y_no_loss[0]

#with open("y_no_loss_data.jsonl", 'w') as f:
#    for entry in y_no_loss:
#        json.dumps(entry, f)
#        f.write('\n')
#y_no_loss = None

# print("done with y no loss")
#

# for i in range(10):
#     file = i
#     with open("x_no_loss_data" + str(i) + ".jsonl", 'w') as f:
#         print("Starting loop:",i)
#         for entry in x_no_loss[i]:
#             json.dumps(entry, f)
#             f.write('\n')
# #


import numpy as np
import itertools
import statistics
import matplotlib.pyplot as plt

def plot_ellsworth_figs(data):

    emotions = ["sadness", "joy", "fear", "anger", "challenge", "boredom", "frustration"]
    vars = ["pleasantness", "attention", "control",
                          "certainty", "anticipated_effort", "responsibility"]

    em_plot = {
        "pleasantness":{x:[] for x in emotions},
        "attention":{x:[] for x in emotions},
        "control":{x:[] for x in emotions},
        "certainty":{x:[] for x in emotions},
        "anticipated_effort":{x:[] for x in emotions},
        "responsibility":{x:[] for x in emotions}
    }

    for x in data:
        for turn in ["turn1", "turn2", "turn3"]:
            emotion = x[turn]["emotion"]
            for var in vars:
                em_plot[var][emotion].append(x[turn]["appraisals"][var])

    for var in em_plot:
        for em in em_plot[var]:
            em_plot[var][em] = statistics.mode(em_plot[var][em])

    fig, ax = plt.subplots()
    ax.scatter(list(em_plot["pleasantness"].values()), list(em_plot["anticipated_effort"].values()))
    ax.set_xlabel("Pleasantness")
    ax.set_ylabel("Effort")
    for i, txt in enumerate(emotions):
        ax.annotate(txt, ( list(em_plot["pleasantness"].values())[i], list(em_plot["anticipated_effort"].values())[i]))
    plt.savefig("results/figs/(mode)effort_vs_pleasantness.png")

    fig, ax = plt.subplots()
    ax.scatter(list(em_plot["responsibility"].values()), list(em_plot["control"].values()))
    ax.set_xlabel("Responsibility")
    ax.set_ylabel("Control")
    for i, txt in enumerate(emotions):
        ax.annotate(txt, ( list(em_plot["responsibility"].values())[i], list(em_plot["control"].values())[i]))
    plt.savefig("results/figs/(mode)control_vs_res.png")

    fig, ax = plt.subplots()
    ax.scatter(list(em_plot["attention"].values()), list(em_plot["certainty"].values()))
    ax.set_xlabel("Attention")
    ax.set_ylabel("Certainty")
    for i, txt in enumerate(emotions):
        ax.annotate(txt, ( list(em_plot["attention"].values())[i], list(em_plot["certainty"].values())[i]))
    plt.savefig("results/figs/(mode)certainty_vs_attention.png")

def plot_confusion_matrix(cm, classes, title, normalize=False, cmap=plt.cm.Blues):
    """
    This function prints and plots the confusion matrix.
    Normalization can be applied by setting `normalize=True`.

    Courtesy of scikit-learn
    """
    if normalize:
        cm = cm.astype("float") / cm.sum(axis=1)[:, np.newaxis]

    plt.imshow(cm, interpolation="nearest", cmap=cmap)
    plt.title(title)
    plt.colorbar()
    tick_marks = np.arange(len(classes))
    plt.xticks(tick_marks, classes, rotation=45)
    plt.yticks(tick_marks, classes)

    fmt = ".2f" if normalize else "d"
    thresh = cm.max() / 2.
    for i, j in itertools.product(range(cm.shape[0]), range(cm.shape[1])):
        plt.text(j, i, format(cm[i, j], fmt),
                    horizontalalignment="center",
                    color="white" if cm[i, j] > thresh else "black")

    plt.ylabel("True label")
    plt.xlabel("Predicted label")
    plt.tight_layout()
    plt.savefig("results/confusion_matrices/" + title + ".png")
    plt.close()


# coding:utf-8
from __future__ import unicode_literals
from __future__ import division
from __future__ import print_function

import tensorflow as tf
import numpy as np
import time
from annoy import AnnoyIndex

from src.data_generator.data_loader import BaseBatch
from src.data_generator.vocabulary import Tokenizer
from src.utils import utils

def init_helper(hparam, question, answer):
    utils.raise_inexistence(question)
    utils.raise_inexistence(answer)
    return dict(
        input_x=utils.read_lines(question),
        input_y=utils.read_lines(answer),
        x_max_len=hparam.x_max_len,
        y_max_len=hparam.y_max_len)

def vector(hparam, model, sess):
    tokenizer = Tokenizer(hparam.vocab_file, segment=hparam.segment)
    batch = BaseBatch(
        tokenizer, init_helper(hparam, hparam.train_q, hparam.train_a))
    saver = tf.train.Saver()
    saver.restore(sess, hparam.model_path)
    return vector_space(hparam, model, sess, batch)

def vector_space(hparam, model, sess, batch):
    features = {'id': 0, 'vecs': []}
    total_size = batch.data_size
    starter = time.time()
    for i in range(total_size // hparam.batch_size + (
            1 if total_size % hparam.batch_size != 0 else 0)):
        fetches, feed_dict = model.infer_step(
            batch.next_batch(hparam.batch_size, features['id']))
        features['id'] += hparam.batch_size
        features['vecs'].append(sess.run(fetches, feed_dict=feed_dict))
        if i % hparam.show_steps == 0 and i:
            speed = hparam.show_steps / (time.time() - starter)
            utils.verbose('step : {:05d} | speed: {:.5f} it/s'.format(i, speed))
            starter = time.time()
    question_vecs = np.reshape(np.array(features['vecs']), [-1, hparam.hidden])[: total_size]
    vec_dim = question_vecs.shape[-1]
    ann = AnnoyIndex(vec_dim)
    for n, ii in enumerate(question_vecs):
        ann.add_item(n, ii)
    ann.build(10)
    return ann

def process(hparam, model, sess):
    ann = vector(hparam, model, sess)
    ann.save(hparam.ann_path)
    utils.verbose('dump annoy into {}'.format(hparam.ann_path))


from socket import *
import os
import time
from time import mktime
from datetime import datetime
from wsgiref.handlers import format_date_time

serverPort = 11600
serverSocket = socket(AF_INET, SOCK_STREAM)
serverSocket.bind(('', serverPort))
serverSocket.listen(1)

def setHeader(x, f = ""):
    #I used https://stackoverflow.com/questions/237079/how-to-get-file-creation-modification-date-times-in-python to find the last modified time. West Point, Ny. 27 September 2017.
    timeMod = "Last-Modified: " + str(format_date_time(mktime(time.localtime(os.path.getmtime(f))))) + "\n"
    #I used https://stackoverflow.com/questions/225086/rfc-1123-date-representation-in-python for the proper HTTP date-time format. West Point, NY. 27 September 2017.
    timeNow = "Date: " + str(format_date_time(mktime(datetime.now().timetuple()))) + "\n"

    connection = "Connection: close\n"

    contentType = "Content-Type: "
    ext = f.split('.')[-1]
    if ext == "html":
        contentType += "text/html\n"
        length = "Content-Length: " + str(len(open(f,'r').read()))
    elif ext == "css":
        contentType += "text/css\n"
        length = "Content-Length: " + str(len(open(f,'r').read()))
    else:
        contentType += "image/jpeg\n"
        length = "Content-Length: " + str(len(open(f,'rb').read()))

    status = "HTTP/1.1 "
    if x == 200:
        status += "200 OK\n"
    elif x == 301:
        status += "301 Moved Permanently\n"
    else:
        status += "404 Not Found\n"
        contentType = "Content-Type: text/html\n"
        header = status + timeNow + conenction + contentType + length
        header += str(len(header))
        return header

    header = status + timeNow + timeMod + connection + contentType + length

    return header

def validRequest(x, files):
    if x == '/' or x == "test1.html":
        return 0
    if x[1:] == "white.html":
        return 2
    if x[1:] == "it105.html":
        return 11
    for i in range(0, len(files)):
        if x[1:] in files[i].replace("\\","/"):
            return i
    return -1

while True:
    connectionSocket, addr = serverSocket.accept()
    message = connectionSocket.recv(1024).decode()
    parsed = message.split('\r\n')
    files = []

    #https://stackoverflow.com/questions/954504/how-to-get-files-in-a-directory-including-all-subdirectories used to look for files in the subdirectories. West Point, NY. 28 September 2017.
    for dirpath, dirnames, filenames in os.walk("./proj1files"):
        for filename in [f for f in filenames]:
            files.append(os.path.join(dirpath, filename))

    try:
        iOfFile = validRequest(parsed[0].split(' ')[1], files)
        if iOfFile == -1:
            header = setHeader(404)
            connectionSocket.send((header + "\n\n" + f).encode())
            conenctionSocket.close()
            continue
        if iOfFile == 1 or iOfFile == 3:
            header = setHeader(301,files[iOfFile])
        else:
            header = setHeader(200,files[iOfFile])
        if "jpg" in files[iOfFile] or "jpeg" in files[iOfFile]:
            f = open(files[iOfFile],'rb').read()
            connectionSocket.send((header + "\n\n").encode() + f)
        else:
            f = open(files[iOfFile],'r').read()
            connectionSocket.send((header + "\n\n" + f).encode())

    except:
        continue

    connectionSocket.close()


"""
Now that all the files are csv, and the headers have been manually edited where they were lacking,
time to combine all the files. Cases to deal with:

    - Header casing is inconsistent
    - Sometimes the name is in one colum, sometimes in two
    - Address isn't often present, but multiple address columns can exist, such as:
      "Billing Address", "Work Address" and "Home Address".
      Address columns are "* Address [1-2].", "* City", "* State", "* Zip", "* Country".
      Variants include "State/Province" and "Postal Code"
    - Keep only most recent address. This means that input files will need to be processed in order.
      Naming is very inconsistent. I manually fixed a few so that the first year is four digits
      and the only numbers are years. The remaining formats include "2019", "2018-2019", "2018-19",
      "2018-9". The important thing for sorting is that anything with a hypen comes last for that
      first year; the default sort order satisfies this.

Target columns are:
    Name - First
    Name - Last
    Email Address
    Phone Number
    Mailing Address
    City
    State
    Zip
    Alternate name
    Alt email
    Alt phone
"""
from glob import glob
import argparse
import re

import numpy as np
import pandas as pd

COLUMNS = ["Name - First", "Name - Last", "Email Address", "Phone Number", "Mailing Address",
           "City", "State", "Zip", "Alternate name", "Alt email", "Alt phone"]

def make_headers_lowercase(df):
    """
    Make all headers lowercase.
    """
    df.columns = [x.lower().strip().strip("*") for x in df.columns]

def make_first_last(df):
    """
    If the dataframe has only "name", split into "First Name" and "Last Name", adding these two
    columns and splitting the name on a space.
    """
    df.columns = [x.replace("given", "first").replace("family", "last") for x in df.columns]
    if "name" in df.columns or "donor" in df.columns:
        nameseries = df["name"] if "name" in df.columns else df["donor"]
        df["first name"] = nameseries.apply(lambda x: x.split(" ")[0])
        df["last name"] = nameseries.apply(lambda x: " ".join(x.split(" ")[1:]))
        return
    else:
        # handle "Donor First Name" and "Donor Last Name"
        df.columns = [x.replace("donor ", "") for x in df.columns]
        if not "first name" in df.columns:
            raise ValueError("No name in header")

def unify_email(df):
    """
    Any header with email in it will be shortened to "email".
    """
    df.columns = ["email" if "email" in x else x for x in df.columns]
    if not "email" in df.columns:
        return
    df["email"] = df["email"].apply(lambda x: x.strip().replace("+AEA-", "@"))
    df["email"] = df["email"].apply(lambda x: "" if not "@" in x else x)

def format_phone(phone):
    """
    Return phone format 000-000-0000.

    Args:
        phone (str) -- phone number in random format
    """
    formatted = re.sub("\D", "", phone)
    if not formatted:
        return ""
    return "-".join([formatted[:3], formatted[3:6], formatted[6:]])

def unify_phone(df):
    """
    Any header with phone in it will be shortened to "phone".

    Returns:
        new df, this cannot make all operations in place
    """
    df.columns = ["phone" if "phone" in x else x for x in df.columns]
    if not "phone" in df.columns:
        return df
    # there is a set of files with some entries containing two phone numbers in a cell
    # these entries are all in the format (000) 000-0000, so create a new row with the second
    # phone number if there are two "(" in the cell. Some entries have text in addition to the
    # numbers, so strip that out. The format function will add in hyphens
    twonums = df["phone"].apply(lambda x: x.count("(") > 1)
    df2 = df[twonums].copy()
    df2["phone"] = df2["phone"].apply(lambda x: re.sub("\D", "", x[15:]))
    df["phone"] = df["phone"].apply(lambda x: x[:15])
    # attempt to unify format
    combined = df.append(df2, ignore_index=True)
    combined["phone"] = combined["phone"].apply(lambda x: format_phone(x))
    return combined

def zip_to_zipcode(df):
    """
    Because zip is special in python, change zip to zip code. Also remove "/postal code".
    """
    df.columns = [x.replace("zip", "zipcode").replace("/postal code", "") for x in df.columns]
    df.columns = [x.replace("postal code", "zipcode") for x in df.columns]
    df.columns = [x.replace("/province", "") for x in df.columns]

def add_addresses(row, table):
    """
    Add addresses from row into table. Names in row should have been unified already with the
    functions above.

    Args:
        row (pd.Series)
        table (Table)
    """
    address_parts = ["address", "city", "state", "zipcode"]
    first = row["first name"]
    last = row["last name"]

    address_names = {x for x in row.index if "address" in x}
    # combine address lines 1 and 2

    add = set()
    remove = set()
    for address_name in address_names:
        if "1" in address_name:
            address2 = address_name.replace("1", "2")
            row[address_name.strip(" 1")] = f"{row[address_name]} {row[address2]}"
            add.add(address_name.strip(" 1"))
            remove = remove.union([address_name, address2])
    address_names -= remove
    address_names = address_names.union(add)

    for address_name in address_names:
        splitname = address_name.split(" ")
        if len(splitname) == 1:
            address = {ap:row[ap] for ap in address_parts}
        else:
            address = {ap:row[" ".join([splitname[0], ap])] for ap in address_parts}
        table.add(first, last, **address)

class Table:
    """
    Rather than try to do a bunch of complicated merging and de-duping with pandas, the entries
    will be added one at a time to the dataframe. Clarity is more important than efficiency for
    this project. This class will manage the merging.

    Alternate names will not be dealt with here, matching will require manual effort. But the
    column is included so at least the final spreadsheet will have it.
    """
    def __init__(self):
        self.data = pd.DataFrame(columns=COLUMNS)

    def __getitem__(self, row_col_tup):
        """
        Index into self.data with a row index or a tuple of (row_index, column_index).
        """
        return self.data.iloc[row_col_tup]

    def __setitem__(self, row_col_tup, val):
        """
        Set value in self.data with a tuple of (row_index, column_index).
        """
        self.data.iloc[row_col_tup] = val

    def add(self, first=None, last=None, email=None, phone=None, address=None, city=None,
            state=None, zipcode=None):
        """
        Add new entry or information to existing entry if the combination of first and last already
        exist. If last exists with a different first, a message will be printed for a record of
        entries to consider merging.

        If an entry with first and last exists, contact information will be compared and added
        where possible. If all slots are full, a message will be printed.
        """
        firstmatch = self.data["Name - First"].apply(lambda s: s.lower()) == first.lower()
        lastmatch = self.data["Name - Last"].apply(lambda s: s.lower()) == last.lower()
        entrymask = firstmatch & lastmatch
        if not sum(entrymask):
            self.data = self.data.append({COLUMNS[0]:first, COLUMNS[1]:last, COLUMNS[2]:email,
                                          COLUMNS[3]:phone, COLUMNS[4]:address, COLUMNS[5]:city,
                                          COLUMNS[6]:state, COLUMNS[7]:zipcode, COLUMNS[8]:None,
                                          COLUMNS[9]:None, COLUMNS[10]:None}, ignore_index=True)
            return
        if sum(entrymask) > 1:
            raise ValueError(f"multiple entries exist for {first} {last}")
        rowi = np.where(entrymask)[0][0]
        if email and email is not None:
            # no email in existing entry, add new one
            if not self[rowi, 2] or self[rowi, 2] is None:
                self[rowi, 2] = email
            # email doesn't match, save as alt if not taken
            elif self[rowi, 2] != email:
                if self[rowi, -2] is None:
                    self[rowi, -2] = email
                # if the alt exists and is same as new one, continue
                elif self[rowi, -2] != email:
                    print(f"Two emails already exist for {first} {last}, cannot add {email}")

        # repeat for phone
        if phone and phone is not None:
            # no phone in existing entry, add new one
            if not self[rowi, 3] or self[rowi, 3] is None:
                self[rowi, 3] = phone
            # phone doesn't match, save as alt if not taken
            elif self[rowi, 3] != phone:
                if self[rowi, -1] is None:
                    self[rowi, -1] = phone
                # if the alt exists and is same as new one, continue
                elif self[rowi, -1] != phone:
                    print(f"Two phone numbers already exist for {first} {last}, cannot add {phone}")

        # add address, newer address will over-write older one
        if address and not address is None:
            self[rowi, 4] = address
            self[rowi, 5] = city
            self[rowi, 6] = state
            self[rowi, 7] = zipcode

def get_files_in_chron_order(csvfiledir):
    """
    Return list of csv file names in chronological order.

    Args:
        csvfiledir (str) -- name of directory containing csv files.
    """
    files = glob(csvfiledir + "/*csv")
    tosort = []
    for f in files:
        stem = f.split("_")[1].rstrip(".csv")
        noletters = re.sub(r"[A-z]", "", stem)
        tosort.append((noletters.strip("-"), f))
    return [f[1] for f in sorted(tosort)]

def main():
    """
    Process all the files and write data to file.
    """
    parser = argparse.ArgumentParser()
    parser.add_argument("csvdir", type=str, help="path to directory containing csv files")
    parser.add_argument("outfile", type=str, help="name, including path, for output csv file")
    args = parser.parse_args()

    table = Table()
    for csvfile in get_files_in_chron_order(args.csvdir):
        print(csvfile)
        curr = pd.read_csv(csvfile)
        curr.fillna("", inplace=True)
        make_headers_lowercase(curr)
        make_first_last(curr)
        unify_email(curr)
        curr = unify_phone(curr)
        zip_to_zipcode(curr)
        for _, row in curr.iterrows():
            email = row["email"] if "email" in row.index else ""
            phone = row["phone"] if "phone" in row.index else ""
            table.add(row["first name"], row["last name"], email.strip(), phone.strip())
            add_addresses(row, table)
    # delete rows with no information other than the name, which tend to be not really people,
    # since spreadsheets have information other than contact information
    # to drop rows, need to use NaN as missing value
    table.data.replace("", np.nan, inplace=True)
    table.data.dropna(how="all", subset=table.data.columns[2:], inplace=True)
    table.data.dropna(how="all", subset=table.data.columns[:2], inplace=True)
    table.data.fillna("")
    table.data.to_csv(args.outfile, index=False)

if __name__ == "__main__":
    main()


# Copyright 2014 Microsoft Corporation
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# Requires Python 2.4+ and Openssl 1.0+
#

from __future__ import print_function

import copy
import glob
import json
import mock
import os
import platform
import random
import subprocess
import sys
import tempfile
import textwrap
import zipfile

from tests.protocol.mockwiredata import *
from tests.tools import *

import azurelinuxagent.common.conf as conf
import azurelinuxagent.common.logger as logger
import azurelinuxagent.common.utils.fileutil as fileutil
import azurelinuxagent.common.version as version

from azurelinuxagent.common.utils.flexible_version import FlexibleVersion
from azurelinuxagent.common.version import *

class TestCurrentAgentName(AgentTestCase):
    def setUp(self):
        AgentTestCase.setUp(self)
        return

    @patch("os.getcwd", return_value="/default/install/directory")
    def test_extract_name_finds_installed(self, mock_cwd):
        current_agent, current_version = set_current_agent()
        self.assertEqual(AGENT_LONG_VERSION, current_agent)
        self.assertEqual(AGENT_VERSION, str(current_version))
        return

    @patch("os.getcwd")
    def test_extract_name_finds_latest_agent(self, mock_cwd):
        path = os.path.join(conf.get_lib_dir(), "{0}-{1}".format(
            AGENT_NAME,
            "1.2.3"))
        mock_cwd.return_value = path
        agent = os.path.basename(path)
        version = AGENT_NAME_PATTERN.match(agent).group(1)
        current_agent, current_version = set_current_agent()
        self.assertEqual(agent, current_agent)
        self.assertEqual(version, str(current_version))
        return

class TestGetF5Platforms(AgentTestCase):
    def test_get_f5_platform_bigip_12_1_1(self):
        version_file = textwrap.dedent("""
        Product: BIG-IP
        Version: 12.1.1
        Build: 0.0.184
        Sequence: 12.1.1.0.0.184.0
        BaseBuild: 0.0.184
        Edition: Final
        Date: Thu Aug 11 17:09:01 PDT 2016
        Built: 160811170901
        Changelist: 1874858
        JobID: 705993""")

        mo = mock.mock_open(read_data=version_file)
        with patch(open_patch(), mo):
            platform = version.get_f5_platform()
            self.assertTrue(platform[0] == 'bigip')
            self.assertTrue(platform[1] == '12.1.1')
            self.assertTrue(platform[2] == 'bigip')
            self.assertTrue(platform[3] == 'BIG-IP')

    def test_get_f5_platform_bigip_12_1_0_hf1(self):
        version_file = textwrap.dedent("""
        Product: BIG-IP
        Version: 12.1.0
        Build: 1.0.1447
        Sequence: 12.1.0.1.0.1447.0
        BaseBuild: 0.0.1434
        Edition: Hotfix HF1
        Date: Wed Jun  8 13:41:59 PDT 2016
        Built: 160608134159
        Changelist: 1773831
        JobID: 673467""")

        mo = mock.mock_open(read_data=version_file)
        with patch(open_patch(), mo):
            platform = version.get_f5_platform()
            self.assertTrue(platform[0] == 'bigip')
            self.assertTrue(platform[1] == '12.1.0')
            self.assertTrue(platform[2] == 'bigip')
            self.assertTrue(platform[3] == 'BIG-IP')

    def test_get_f5_platform_bigip_12_0_0(self):
        version_file = textwrap.dedent("""
        Product: BIG-IP
        Version: 12.0.0
        Build: 0.0.606
        Sequence: 12.0.0.0.0.606.0
        BaseBuild: 0.0.606
        Edition: Final
        Date: Fri Aug 21 13:29:22 PDT 2015
        Built: 150821132922
        Changelist: 1486072
        JobID: 536212""")

        mo = mock.mock_open(read_data=version_file)
        with patch(open_patch(), mo):
            platform = version.get_f5_platform()
            self.assertTrue(platform[0] == 'bigip')
            self.assertTrue(platform[1] == '12.0.0')
            self.assertTrue(platform[2] == 'bigip')
            self.assertTrue(platform[3] == 'BIG-IP')


import Parser
import BuildPath
import FindFile
import os

def calcIndex(file1,file2, Fc, filewrite):

    release1=set(open(file1,'r'))
    release2=set(open(file2,'r'))

    filewrite.write("###########################################\n")
    filewrite.write("Indici di "+file1.__str__()+" "+file2.__str__()+" :\n")
    unionClass=release1.union(release2)
    interClass=release1.intersection(release2)
    filewrite.write("Classi totali tra le due release="+str(len(unionClass))+"\n")
    filewrite.write("Intersezione tra le classi delle due release:"+str(len(interClass))+"\n")
    Mt = len(release1)
    Fa = len(release2-interClass)
    Fd = len(release1 - interClass)
    filewrite.write("Mt=" + str(Mt) + "\n")
    filewrite.write("Mt2=" + str(len(release2)) + "\n")
    filewrite.write("Fa="+str(Fa)+"\n")
    filewrite.write("Fd="+str(Fd)+"\n")
    filewrite.write("Fc="+str(Fc) +"\n")
    filewrite.write("SMI="+str((Mt-(Fa+Fc+Fd))/Mt) + "\n")

def diffClasses(release1, release2):

    numDiff = 0

    classListPath = projectPath + "num_Class\\"
    classList = os.listdir(classListPath)[numFile]

    fileClass = open(classListPath + classList, 'r')

    for line in fileClass:
        pureLine = line.strip()
        file1 = FindFile.findFile(pureLine + ".*", release1)
        file2 = FindFile.findFile(pureLine + ".*", release2)

        if file1 is not None and file2 is not None:
            with open(file1, encoding="utf8", errors='ignore') as f:
                t1 = f.read().splitlines()
                t1s = set(t1)

            with open(file2, encoding="utf8", errors='ignore') as f:
                t2 = f.read().splitlines()
                t2s = set(t2)

            findDiff = False

            print(file1)
            print(file2)
            # in file1 but not file2
            print("Only in file1")
            for diff in t1s - t2s:
                findDiff = True
                print(t1.index(diff), diff)

            if findDiff is False:
                # in file2 but not file1
                print("Only in file2")
                for diff in t2s - t1s:
                    findDiff = True
                    print(t2.index(diff), diff)

            if findDiff:
                numDiff += 1
        else:
            print("il file non è presente in entrambe le release")

    print(numDiff)
    return numDiff

listFile=[]
dirIndex="dirIndex\\"
projectList = os.listdir(BuildPath.ROOT_PATH)
for project in projectList:
    projectPath = BuildPath.ROOT_PATH + project +"\\"
    dir=projectPath+Parser.DIR_CLASS
    path_dirindex=BuildPath.create_directory(projectPath,dirIndex)
    fileparsed = os.listdir(dir)
    fileIndex=BuildPath.create_file(path_dirindex,"fileIndex.txt")

    releasePath = projectPath + "release\\"
    releaseList = os.listdir(releasePath)

    numFile=0
    while numFile<len(fileparsed):
        if numFile+1<len(fileparsed):
            Fc = diffClasses(releasePath + releaseList[numFile], releasePath + releaseList[numFile + 1])
            calcIndex(dir+fileparsed[numFile],dir+fileparsed[numFile+1], Fc, fileIndex)

        numFile+=1