#encoding: utf-8
"""
@project = PythonScriptMergeTool
@file = DoConfMgr
@author = caigh
@Create_time = 2018/3/30 20:41
"""
import yaml
import os
import io

class DevOpsConfMgr:

    def __init__(self):
        curPath = os.path.dirname(os.path.abspath(__file__))
        logpath = os.path.dirname(os.path.dirname(curPath)) + '\\conf\\'
        self.outPutPath = os.path.dirname(os.path.dirname(curPath)) + '\\output\\data\\'
        self.inputPath = os.path.dirname(os.path.dirname(curPath)) + '\\input\\'
        self.confPath = logpath
        self.commConfFileName = 'whalecloud_comm.cfg'
        self.sqlUpdateConfigFileName = 'whalecloud_update_sql.cfg'
        self.Excel2DBConfigFileName = 'Excel2DB.cfg'
        self.checkTableFileName = 'checkTable.cfg'
        self.ruleMap = {}
        self.logLevel = 'ERROR'  #logging.ERROR
        self.TestFlag = 'N'
        self.dbNameMap = {'DBConnect': 'ODBC',
                          'DBCRMConnect': 'CRMODBC'}
        self.midTableNameList = []
        self.dbConnMap = {}

    def loadAllConfigureFile(self):
        fileName = self.confPath + self.commConfFileName
        self.commConfig = yaml.safe_load(io.open(fileName, mode='r', encoding='utf-8'))
        fileName = self.confPath + self.sqlUpdateConfigFileName
        self.sqlUpdateStrMap = yaml.safe_load(io.open(fileName, mode='r', encoding='utf-8'))
        self.sqlSelectStrMap = self.sqlUpdateStrMap['SELECT']
        fileName = self.confPath + self.Excel2DBConfigFileName
        self.excelControlRuleMap = yaml.safe_load(io.open(fileName, mode='r', encoding='utf-8'))
        fileName = self.confPath + self.checkTableFileName
        self.checkTableList = yaml.safe_load(io.open(fileName, mode='r', encoding='utf-8'))

    def setCommConf(self):
        if self.commConfig.get('LogLevel'):
            logLevel = self.commConfig['LogLevel'].split('=')
            logLevel = logLevel[1].replace(' ', '')
            self.logLevel = logLevel

        if self.commConfig.get('DataBase'):
            for dbconn in self.commConfig['DataBase']:
                dbconn = dbconn.split('=')
                dbSid = dbconn[1].strip()
                dbName = dbconn[0].strip()
                self.dbConnMap[self.dbNameMap[dbName]] = dbSid

    def getTempletFileName(self):
        if self.commConfig.get('TempletFileName'):
            return self.commConfig['TempletFileName']
        return None

    def getNewTempletFileName(self):
        if self.commConfig.get('NewTempletFileName'):
            return self.commConfig['NewTempletFileName']
        return None

    def getTempletSheetList(self):
        if self.commConfig.get('TempletSheetList'):
            return self.commConfig['TempletSheetList']
        return None

    def getTempletTile(self):
        if self.commConfig.get('TempletTile'):
            return self.commConfig['TempletTile']
        return []

    def getColumnMap(self):
        if self.commConfig.get('ColumnMap'):
            return self.commConfig['ColumnMap']
        return []

    def getOfferCodeMap(self):
        if self.commConfig.get('OfferCodeMap'):
            return self.commConfig['OfferCodeMap']
        return []

    def getOfferFamilyMap(self):
        if self.commConfig.get('FamilyMap'):
            return self.commConfig['FamilyMap']
        return []

    def getSheetNum(self):
        if self.commConfig.get('sheetNum'):
            return self.commConfig['sheetNum']
        return 1

    def getFileName(self, fileTag='FileName'):
        if self.commConfig.get(fileTag):
            return self.commConfig[fileTag]
        return None


from django.contrib.postgres.indexes import BrinIndex
from django.db import models
from django.utils.translation import gettext_lazy as _

from applications.accounts.models import User
from shared.models.transaction import AbstractTransaction

__all__ = 'Transaction',

class Transaction(AbstractTransaction):
    purpose_model = User

    purpose = models.ForeignKey(
        purpose_model,
        on_delete=models.PROTECT,
        related_name='transactions',
        verbose_name=_("Purpose")
    )

    class Meta:
        ordering = ('-created_at',)
        verbose_name = _('transaction')
        verbose_name_plural = _('transactions')
        indexes = (
            BrinIndex(fields=('created_at',)),
        )


# -*- coding: utf-8 -*-
import scrapy
import re

from imdb.pipelines import ImdbPipeline
from imdb.items import DirectorItem

imdbpipeline = ImdbPipeline()

def get_full_url(actor_id):
    base_url = "https://www.imdb.com/title/{}/"
    return base_url.format(actor_id)

def prepare_movie_urls():
    """
        准备movie url
    """
    search_result = imdbpipeline.get_actor_id(type_id=2)
    base_url = "https://www.imdb.com/name/{}/"
    urls = []
    for result in search_result:
        title = result[0]
        url = base_url.format(title)
        urls.append(url)
    return urls

class DirectorScoreSpider(scrapy.Spider):
    name = 'director_score'
    allowed_domains = ['imdb.com']
    base_url = 'https://www.imdb.com'
    start_urls = prepare_movie_urls()
    base_title = '/title/{}/'
    def parse(self, response):
        # ---------------------------------------------------------------------------------
        # 获取导演作品集
        try:
            titles = re.findall(r'id="director-(.*?)"', response.text)
            productions = response.xpath('//div[@id="filmo-head-director"]/preceding-sibling::div[1]/div/b/a/@href').extract()
            if not productions:
                titles = re.findall(r'id="director-(.*?)"', response.text)
                for title in titles:
                    production = self.base_title.format(title)
                    productions.append(production)
        except Exception as e:
            print('productions_set Error, ',e)
            productions = []
        # ---------------------------------------------------------------------------------
        # 获取演员名字
        try:
            actor_name = response.xpath('//span[@class="itemprop"]/text()').extract()[0]
        except Exception as e:
            print('actior_name Error, ', e)
            actor_name = ''
        print(actor_name)
        # ---------------------------------------------------------------------------------

        if len(productions) == 0:
            pass
        else:
            actor_item = DirectorItem()
            actor_item['id'] = self.extract_actor_id_from_url(response.url)[:-1]
            actor_item['director_name'] = actor_name

            for production in productions:
                production_url = self.base_url+production
                score = imdbpipeline.get_movie_item(actor_item['id'])
                if score != -1:
                    # 首先尝试在movie数据库中寻找，如果不存在，在请求该网页。
                    actor_item['movie_score'] = score
                    actor_item['movie_num'] = movie_num
                    actor_item['director_score'] = 0
                    yield actor_item
                else:
                    yield scrapy.Request(production_url,
                        callback=self.get_movie_score,
                        meta={'actor_item':actor_item})

    def get_movie_score(self, response):
        actor_item = response.meta['actor_item']

        # ---------------------------------------------------------------------------------
        # 获取电影评分
        try:
            movie_score = response.xpath(r'//span[@itemprop="ratingValue"]/text()').extract()[0]
            movie_num = 1
        except Exception as e:
            print('movie score Error, ', e)
            movie_score = '0'
            movie_num = 0

        actor_item['movie_score'] = movie_score
        actor_item['movie_num'] = movie_num
        actor_item['director_score'] = 0
        yield actor_item

    def extract_actor_id_from_url(self, actor_url):
        if actor_url is None:
            return None
        return re.search(r'(nm[0-9]{7}.*?)/', actor_url).group(0)


'''Node.py: This class implements simple node object and associated methods.
 Nodes are created by passing in an x,y cartesian location for the node.  Node labels are created
 by just assigning letters to nodes in ascending ASCII order; current index is kept as class variable
 to ensure that each new node has a unique label.
 The most interesting  internal method here is makeLabel, which generates unique labels.  We want easy to read
 alpha labels...so it basically counts up in Base26, generating the corresponding ASCII labels.  Tricky!
 '''

__author__ = "Eck Doerry"
__copyright__ = "Copyright 2018, Northern Arizona University, Flagstaff AZ"

import scipy.spatial as graph
import numpy as np

class Node:

# CLASS VAR:  curIndex is used to make sure that all nodes have a unique label.  Incremented every time a new
#  node is generated.
    curIndex=1

    # CLASS METHOD.  Allows for reseting the index counter for generating new nodes. Used only for debugging!
    def reset(cls):
        cls.curIndex=1

    # Create a new node at point x-y.  Optionally pass in a label for the node.  If no explicit label is passed
    # in (which is usually the case), it auto-generates the next label in a sequences starting with A-Z, then
    # going to AA, AB, AC, etc.
    def __init__(self,newX,newY, newLabel= '*'):
        self.x=newX
        self.y=newY
        if newLabel=='*':
            self.label=self.makeLabel()
        else:
            self.label=newLabel

    # Makes an upper case label starting at 'A'.  If >26 nodes, goes to 'AB', 'AC', etc.
    # Works by basically counting in Base26:  converts the index into strings in Base26!
    def makeLabel(self):
        index=self.curIndex
        out = []
        while index>25:
            out.append(index % 26)
            index = int(index / 26)

        out.append(index-1)
        out.reverse()
        Node.curIndex +=1
        return "".join([chr(65+x) for x in out])

    # Distance:  returns the cartesian distance between this node some other node that is passed in.
    def distance(self,aNode):
        return graph.distance.euclidean([self.x,self.y],[aNode.x,aNode.y])

    def get(self):
        return ([self.x,self.y],self.label)

# # Unit tests
# x=Node(4,5)
# y=Node(1,2)
# print(x.get())
# print(y.get())
# print(x.distance(y))


'''
Prepare fields for Lawrence plots
- 3D fields for mass weighted OH
- 3D fields for reaction flux OH+CH4

Put data into boxes that correspond to those in Lawrence et al., 2001
'''
#**********************************************************
import netCDF4 as ncdf
import numpy as np
import tracer_var_codes as vcode

def lawr(jobid,nyrs):
    '''
OH mass weighted
output: 3D field, time average
    '''
    file    = ncdf.Dataset('/scratch/ih280/netscratch/um/'+jobid+'/'+jobid+'_oh.nc')
    oh      = file.variables['OH'][-12*nyrs:]
    oh      = np.mean(oh,axis=0)
    file.close()
    file    = ncdf.Dataset('/scratch/ih280/netscratch/um/'+jobid+'/'+jobid+'_P-theta.nc')
    P       = file.variables['p'][-12*nyrs:]
    P       = np.mean(P,axis=0)
    file.close()
    file    = ncdf.Dataset('/scratch/ih280/netscratch/um/'+jobid+'/'+jobid+'_airmass.nc')
    airmass = file.variables['airmass_atm'][-12*nyrs:]
    airmass = np.mean(airmass,axis=0)
    return (oh,P,airmass)

def boxes(pres, data_zm, mass_zm, zero, lat, hb, lb):
    '''
Distribute data into boxes
creates a mean box value for data, airmass and a zero variable
    '''
    # pbox: list of 12 lists for the 12 atm boxes
    # nb of vert boxes * nb of horiz boxes
    # The first step will result in a nested array which then has to be flattened.
    pbox1    = []
    pbox2    = []
    pbox3    = []
    # boxes: list of 12 lists for the 12 atm boxes
    # nb of vert boxes * nb of horiz boxes
    # Will contain flattened data from pbox1
    databox  = pbox1[:]
    massbox  = pbox2[:]
    zerobox  = pbox3[:]
    for i in range(len(hb)*(len(lb)-1)):
      pbox1.append([])
      pbox2.append([])
      pbox3.append([])
      databox.append([])
      massbox.append([])
      zerobox.append([])
    # Devide atmosphere into vertical boxes, spanning all latitudes
    # z contains indicies as a function of latitude for all pressure levels:
    # | z[2] | hb[1] > P >= hb[2]
    # | z[1] | hb[0] > P >= hb[1]
    # | z[0] |         P >= hb[0]
    z        = [[],[],[]]
    for i in range(len(lat)):
      z[0].append (np.where(  pres[:,i]>=hb[0] )                     [0])
      z[1].append (np.where(( pres[:,i]>=hb[1] ) & (pres[:,i]<hb[0]))[0])
      z[2].append (np.where(( pres[:,i]>=hb[2] ) & (pres[:,i]<hb[1]))[0])
    # Put data into the boxes.
    # This will result in a nested array, that then has to be flattened.
    # Use indices of z for the height boundaries.
    # Create latitude boundaries in loop.
    for i in range(len(lb)-1):              # number of lat boxes
      latlow  = np.where (lat >= lb[i]   )[0][ 0]
      lathigh = np.where (lat <  lb[i+1] )[0][-1]
      for l in range(latlow,lathigh+1):   # lat slice
        for h in range(len(hb)):        # number of vertical boxes
            pbox1[i*3+h].append (list( data_zm [z[h][l], l] ))
            pbox2[i*3+h].append (list( mass_zm [z[h][l], l] ))
            pbox3[i*3+h].append (list( zero    [z[h][l], l] ))
    # Flatten lists in pbox1
    for i in range(len(hb)*(len(lb)-1)):
      a=[]
      b=[]
      c=[]
      for k in pbox1[i]:
        for l in k:
            a.append(l)
      for k in pbox2[i]:
        for l in k:
            b.append(l)
      for k in pbox3[i]:
        for l in k:
            c.append(l)
      databox [i] = np.array( a[:] ) # OH
      massbox [i] = np.array( b[:] ) # Mass
      zerobox [i] = np.array( c[:] ) # Zeros
    del a, b, c, pbox1, pbox2, pbox3
    return (databox, massbox, zerobox)


from pathlib import Path

from ray import tune
from ray.util.sgd.utils import BATCH_SIZE

import workloads.common as com
from fluid.trainer import TorchTrainer
from workloads.common import cifar as workload

DATA_PATH, RESULTS_PATH = com.detect_paths()
EXP_NAME = com.remove_prefix(Path(__file__).stem, "tune_")

def create_grid_search_space(exp_no):
    #     mutations = space.create_grid_space_1()
    method_name = "space.create_grid_space_" + str(exp_no)
    mutations = eval(method_name)()
    return {key: tune.sample_from(val) for key, val in mutations.items()}

def setup_tune_scheduler(exp_no):
    search_space = create_grid_search_space(exp_no)
    sync_to_driver = not RESULTS_PATH.startswith("/nfs")

    return dict(
        config=search_space,
        resources_per_trial={"gpu": 1},
        sync_to_driver=sync_to_driver,
        local_dir=RESULTS_PATH,
        name=EXP_NAME + str(exp_no),
    )

def main():
    exp_no, sd = com.init_ray()
    MyTrainable = TorchTrainer.as_trainable(
        data_creator=workload.data_creator,
        model_creator=workload.model_creator,
        loss_creator=workload.loss_creator,
        optimizer_creator=workload.optimizer_creator,
        config={"seed": sd, BATCH_SIZE: 64, "extra_fluid_trial_resources": {}},
    )

    params = {
        # **com.run_options(__file__),
        # 'stop': workload.create_stopper(),
        **setup_tune_scheduler(exp_no),
    }

    analysis = tune.run(MyTrainable, stop={"training_iteration": 81}, **params)

    dfs = analysis.trial_dataframes
    for logdir, df in dfs.items():
        ld = Path(logdir)
        df.to_csv(ld / "trail_dataframe.csv")

if __name__ == "__main__":
    main()


import sys, getopt, os, pickle
import numpy as np
import tensorflow as tf
import librosa

sys.path.insert(0,'..')
from global_vars import subdirectory_list, n_fft, n_mels, t, hop_length, sr, genres
from shared_func import print_and_exit

help_msg = 'data_preprocessing.py -d <dir>, where dir is the directory containing labeled sub-dirs in which the .wav files are located'

# The following functions can be used to convert a value to a type compatible
# with tf.Example. https://www.tensorflow.org/tutorials/load_data/tfrecord#tfexample

def _bytes_feature(value):
  """Returns a bytes_list from a string / byte."""
  if isinstance(value, type(tf.constant(0))):
    value = value.numpy() # BytesList won't unpack a string from an EagerTensor.
  return tf.train.Feature(bytes_list=tf.train.BytesList(value=[value]))

def _float_feature(value):
  """Returns a float_list from a float / double."""
  return tf.train.Feature(float_list=tf.train.FloatList(value=[value]))

def _int64_feature(value):
  """Returns an int64_list from a bool / enum / int / uint."""
  return tf.train.Feature(int64_list=tf.train.Int64List(value=[value]))

def add_padding(array):
	padded = np.zeros((sr * 10))
	padded[0:array.shape[0]] = array

	return padded

#https://www.tensorflow.org/tutorials/load_data/tfrecord#creating_a_tfexample_message
def spectogram_to_example(spectogram, genre):
	feature = {
		'label': _int64_feature(genre),
		'spectogram': _bytes_feature(spectogram),
	}
	return tf.train.Example(features = tf.train.Features(feature = feature))

def create_spectogram(dir_to_browse, writer, genre):
	files = os.listdir(dir_to_browse)

	for file in files:
		y, _ = librosa.load(dir_to_browse + '/' + file, sr = sr)
		if(y.shape[0] < (10 * sr)):
			y = add_padding(y)
		spectogram = librosa.feature.melspectrogram(y = y, sr=sr, n_fft=n_fft, hop_length=hop_length, n_mels=n_mels)
		spectogram_pwr = librosa.power_to_db(spectogram, ref = np.max)

		spectogram_1d = np.reshape(spectogram_pwr, (n_mels * t)).tobytes()
		example = spectogram_to_example(spectogram_1d, genre)
		writer.write(example.SerializeToString())

#what if we already have the pickle but have updated the directory with the files -- this wont work correctly
# better to first create the pickle and then compare with old one if such exists
def maybe_create_spectograms(main_dir):
	genre_map = dict((label, idx) for idx, label in enumerate(genres))

	for subdir in subdirectory_list:
		curr_tf_record_path = main_dir + '/' + subdir + '.tfrecord'
		curr_subdir_path = main_dir + '/' + subdir
		if os.path.isfile(curr_tf_record_path):
			print(curr_tf_record_path, "already exists... skipping")
			continue

		print("Writing to ", curr_tf_record_path, "... ", end = "", flush = True)
		writer = tf.io.TFRecordWriter(path = curr_tf_record_path)
		for genre in genres:
			dir_to_browse = curr_subdir_path + '/' + genre
			create_spectogram(dir_to_browse, writer, genre_map[genre])
		print("done")

def main(argv):
	main_dir = ""

	try:
		opts, _ = getopt.getopt(argv,"h:d:",["main_dir="])
	except:
		print_and_exit(help_msg, 2)

	for opt, arg in opts:
		if opt == '-h':
			print_and_exit(help_msg, 0)
		elif opt == '-d':
			main_dir = arg

	if main_dir == "" :
		print_and_exit(help_msg, 2)

	maybe_create_spectograms(main_dir)

if __name__ == "__main__":
	main(sys.argv[1:])


import os
import warnings

import gym

from sacking.environment import load_env
from sacking.policy import DiscretePolicy
from sacking.q_network import DiscreteQNetwork
from sacking.trainer import train

def main():
    if (os.environ.get('OMP_NUM_THREADS') != '1'
           or os.environ.get('MKL_NUM_THREADS') != '1'):
        warnings.warn('running without OMP_NUM_THREADS=1 MKL_NUM_THREADS=1')

    env = load_env('gym/CartPole-v1')
    valid_env = load_env('gym/CartPole-v1')

    assert isinstance(env.observation_space, gym.spaces.Box)
    assert len(env.observation_space.shape) == 1
    obs_dim = env.observation_space.shape[0]

    assert isinstance(env.action_space, gym.spaces.Discrete)
    assert len(env.action_space.shape) == 0
    action_dim = env.action_space.n

    policy = DiscretePolicy(obs_dim, action_dim,
                            hidden_layers=[64, 64])
    q_network = DiscreteQNetwork(obs_dim, action_dim,
                                 hidden_layers=[64, 64],
                                 num_nets=2)

    train(
        policy, q_network, env,
        batch_size=128,
        learning_rate=0.003,
        num_steps=50000,
        replay_buffer_size=100000,
        rundir='output',
        validation_env=valid_env,
    )

if __name__ == '__main__':
    main()


__author__ = 'christopherdavidson'

import uuid
from common.database import Database
from models.room import Room

####################################
# ROOM MATRIX: Keeps Track of Rooms Available for meetings
####################################

class RoomMatrix(object):
    counter = 2     # shared for all instances

    # CONSTRUCTOR
    def __init__(self, roomNum=None, room_id=None, _id=None):
        RoomMatrix.counter += 1
        self.roomNum = RoomMatrix.counter
        self.room_id = room_id
        self._id = uuid.uuid4().hex if _id is None else _id

    # STORED DATA IN MEETING CLASS
    def json(self):
        return {
            'roomNum': self.roomNum,
            'room_id': self.room_id,
            '_id': self._id
        }

    #### INSTANCE METHODS ####
    def create_room(self):
        room = Room(meetings=None,roomNum=self.roomNum)
        self.room_id = room._id
        room.save_to_mongo()
        Database.insert(collection='office', data=self.json())
        return room._id

    #### CLASS METHODS ####
    @classmethod
    def get_room_number(cls, roomID):
        office = Database.find_one('office', {'room_id': roomID})
        r_number = office['room']
        return r_number

    @classmethod
    def get_rooms(cls):
        return [room for room in Database.find(collection='office', query={})]

    @classmethod
    def get_by_id(cls, m_id):
        office = Database.find_one('office', {'_id': m_id})
        r_id = office['room_id']
        return r_id

    @classmethod
    def delete_room(cls, office_id, room_id):
        if room_id is not None:
            # room.from_mongo(by_id) should return a class object
            # so we can use (.) operator to access members of room
            room = Room.get_from_mongo(room_id)
            # need to make sure we don't delete room and its corresponding matrix_id
            # if there are existing meetings in the room object; else don't delete either object
            if len(room.meetings) < 1:
                if RoomMatrix.counter > 2:
                    RoomMatrix.counter -= 1
                room.delete_room_base(room_id)
                Database.remove_one(collection='office', searchVal=office_id)
                return True
        return False


"""
Graph Attention Networks (PPI Dataset) in DGL using SPMV optimization.
Multiple heads are also batched together for faster training.
Compared with the original paper, this code implements
early stopping.
References
----------
Paper: https://arxiv.org/abs/1710.10903
Author's code: https://github.com/PetarV-/GAT
Pytorch implementation: https://github.com/Diego999/pyGAT
"""

import os
import numpy as np
import torch
import dgl
import json
import torch.nn.functional as F
import argparse
import torch.nn as nn
from data.ShockGraphDataset_fsl import *
from models.embedding_network import Classifier
from torch.utils.data import DataLoader
from functools import partial

def all_pairwise_similarity(X1,X2,distance='l2'):

    dot_product=torch.matmul(X1,torch.t(X2))
    if distance == 'l2':
        a=torch.sum(torch.mul(X1,X1),1)
        b=torch.sum(torch.mul(X2,X2),1)
        ab=dot_product
        D=a.unsqueeze_(1)-2*ab+b.unsqueeze_(0)
        return -D
    else:
        a=torch.sqrt(torch.sum(torch.mul(X1,X1),1))
        b=torch.sqrt(torch.sum(torch.mul(X2,X2),1))
        ab=dot_product
        dem=torch.ger(a,b)
        return torch.div(ab,dem)

def predict(D,labels):

    classes = torch.unique_consecutive(labels)
    d2c=torch.zeros(D.shape[0],len(classes),dtype=torch.float32)
    nn_classes=torch.zeros(D.shape[0],dtype=torch.int32)

    for idx in range(d2c.shape[0]):
        sub_D=D[idx,:]
        class_distances=torch.zeros(len(classes))
        for jdx in range(len(classes)):
            class_exemplars=torch.where(labels==classes[jdx])
            start=class_exemplars[0][0]
            stop=class_exemplars[0][-1]+1
            max_value=torch.max(sub_D[start:stop])
            class_distances[jdx]=max_value

        d2c[idx,:]=class_distances
        nn_classes[idx]=classes[torch.argmax(class_distances)]

    return d2c,nn_classes

def predict_nbnn(D,labels,nodes_per_graph,samples):

    classes = torch.unique_consecutive(labels)
    d2c=torch.zeros(samples,len(classes),dtype=torch.float32)
    nn_classes=torch.zeros(samples,dtype=torch.int32)

    beg=0
    end=nodes_per_graph[0]
    for idx in range(d2c.shape[0]):
        sub_D=D[beg:end,:]

        class_distances=torch.zeros(len(classes))
        for jdx in range(len(classes)):
            class_exemplars=torch.where(labels==classes[jdx])
            start=class_exemplars[0][0]
            stop=class_exemplars[0][-1]+1
            max_values,indices=torch.max(sub_D[:,start:stop],dim=1)
            class_distances[jdx]=torch.sum(max_values)

        d2c[idx,:]=class_distances
        nn_classes[idx]=classes[torch.argmax(class_distances)]

        if idx < d2c.shape[0]-1:
            beg=beg+nodes_per_graph[idx]
            end=beg+nodes_per_graph[idx+1]

    return d2c,nn_classes

def main(args):

    if args.gpu<0:
        device = torch.device("cpu")
    else:
        device = torch.device("cuda:" + str(args.gpu))

    # create dataset
    config_file=json.load(open(args.cfg))
    train_dir=config_file['train_dir']
    test_dir=config_file['test_dir']
    dataset=config_file['dataset']
    cache_io=config_file['cache_io']
    napp=config_file['node_app']
    eapp=config_file['edge_app']
    symm_io=config_file['symm_io']
    shuffle_io=config_file['shuffle_io']
    n_classes=config_file['num_classes']
    apply_da=config_file['data_augment']
    rad_scale=config_file['rad_scale']
    angle_scale=config_file['angle_scale']
    length_scale=config_file['length_scale']
    curve_scale=config_file['curve_scale']
    poly_scale=config_file['poly_scale']
    bdir=os.path.basename(train_dir)

    input_dim=58
    if napp:
        input_dim=input_dim+21

    if eapp:
        input_dim=input_dim+9

    norm_factors={'rad_scale':rad_scale,'angle_scale':angle_scale,'length_scale':length_scale,'curve_scale':curve_scale,'poly_scale':poly_scale}

    prefix='data-'+str(bdir)+':'+'mign'+'_m-tag_ni-'+str(input_dim)+'_nh-'+str(args.n_hidden)+'_lay-'+str(args.n_layers)+'_napp-'+str(napp)+'_eapp-'+str(eapp)+'_ro-'+str(args.readout)
    prefix+='_loc-'+str(args.local)+'_nshot-'+str(args.n_shot)+'_kway-'+str(args.k_way)+'_samp-'+str(args.samples)+'_epi-'+str(args.episodes)+'_norm-'+str(args.norm)+'_dist-'+str(args.dist)

    if args.local == False:
        prefix += '_emb-'+str(args.embed_dim)

    if args.readout == 'spp':
        extra='_ng-'+str(args.n_grid)
        prefix+=extra

    print('saving to prefix: ', prefix)
    test_episodes=600
    # create train dataset
    fsl_dataset=ShockGraphDataset(test_dir,dataset,norm_factors,n_shot=args.n_shot,k_way=args.k_way,episodes=test_episodes,test_samples=args.samples,
                                  node_app=napp,edge_app=eapp,cache=True,symmetric=symm_io,data_augment=False,grid=args.n_grid)

    model_files=glob.glob(prefix+'*pth')
    model_files.sort()

    numb_train=args.n_shot*args.k_way

    for state_path in model_files:
        print('Using weights: ',state_path)

        model = Classifier(input_dim,
                           args.n_hidden,
                           args.embed_dim,
                           args.n_layers,
                           args.hops,
                           args.readout,
                           None,
                           args.dropout,
                           args.local,
                           args.norm,
                           args.n_grid,
                           args.K,
                           device)

        model.load_state_dict(torch.load(state_path)['model_state_dict'])
        model.to(device)
        model.eval()

        class_accuracy=np.zeros(test_episodes)
        for idx in tqdm(range(test_episodes)):

            bg,label=fsl_dataset[idx]

            embeddings = model(bg)

            nodes_per_graph=bg.batch_num_nodes

            if args.local and args.readout == 'spp':
                nodes_per_graph=[args.n_grid*args.n_grid]*bg.batch_size

            if args.local:
                support_exemplars=np.sum(nodes_per_graph[:numb_train])
            else:
                support_exemplars=numb_train

            train_embeddings=embeddings[:support_exemplars,:]
            if args.local:
                train_labels=np.repeat(label[:numb_train],nodes_per_graph[:numb_train])
            else:
                train_labels=label[:numb_train]

            test_embeddings=embeddings[support_exemplars:,:]
            ground_truth=label[numb_train:]
            _,test_labels=torch.unique_consecutive(label[numb_train:],return_inverse=True)

            D=all_pairwise_similarity(test_embeddings,train_embeddings,args.dist)

            if args.local:
                prediction,nn_classes=predict_nbnn(D,train_labels,nodes_per_graph[numb_train:],test_labels.shape[0])
            else:
                prediction,nn_classes=predict(D,train_labels)

            acc=torch.sum(nn_classes==ground_truth)/float(len(ground_truth))
            class_accuracy[idx]=acc

        print('Class Accuracy:{:4f}%'.format(np.mean(class_accuracy)*100.0))
        del model

if __name__ == '__main__':
    parser = argparse.ArgumentParser(description='GraphTAGConv and GraphSGConv')

    parser.add_argument("--cfg", type=str, default='cfg/stl10_00.json',
                        help="configuration file for the dataset")
    parser.add_argument("--gpu", type=int, default=-1,
                        help="which GPU to use. Set -1 to use CPU.")
    parser.add_argument("--dropout", type=float, default=0.5,
                        help="dropout probability")
    parser.add_argument("--lr", type=float, default=1e-2,
                        help="learning rate")
    parser.add_argument("--n-hidden", type=int, default=512,
                        help="number of hidden gsage units")
    parser.add_argument("--n-layers", type=int, default=2,
                        help="number of hidden gcn layers")
    parser.add_argument("--weight-decay", type=float, default=5e-4,
                        help="Weight for L2 loss")
    parser.add_argument("--readout", type=str, default="mean",
                        help="Readout type: mean/max/sum")
    parser.add_argument("--hops", type=int, default=2,
                        help="number of hops")
    parser.add_argument("--n-grid", type=int, default=8,
                        help="number of grid cells")
    parser.add_argument("--K", type=float, default=100,
                        help="sort pooling keep K nodes")
    parser.add_argument("--embed_dim", type=int, default="192",
                        help="dim to embed")
    parser.add_argument("--n_shot", type=int, default=5,
                        help="N shot per class")
    parser.add_argument("--k_way", type=int, default=5,
                        help="K way classes")
    parser.add_argument("--episodes", type=int, default=600,
                        help="number of episodes")
    parser.add_argument("--samples", type=int, default=15,
                        help="samples")
    parser.add_argument("--dist", type=str,default='cos',
                        help="should i use l2/coss for distance? ")
    parser.add_argument("--norm", type=bool, default=False,
                        help="normalize or not")
    parser.add_argument("--local", type=bool, default=False,
                        help="do nbnn classify")
    parser.add_argument("--steps", type=int, default=3,
                        help="steps to optimize")

    args = parser.parse_args()
    print(args)

    main(args)


# -*- coding: utf-8 -*-
"""
Created on Tue May 12 10:08:59 2020
"""

import io

QUOTE = {
        "'":"'",
        '"':'"',
        }
BRACKETS = {'(': ')'}
BRACKETS_END = {')':1}
SPACE = {' ':1, '\t':1, '\r':1, '\n':1}

def quoteData(content, index):
    data = ""
    q_e = QUOTE[content[index]]
    c_len = len(content)
    escape = False
    index+=1;
    while index < c_len:
        c = content[index]
        if c == '\\':
            escape = True
        elif c == q_e:
            if(not escape):
                return data, index+1
            else:
                data += c;
        else:
            escape = False
            data += c
        index+=1
    return data, index+1

def normalData(content, index):
    data = ""
    c_len = len(content)
    while index < c_len:
        c = content[index]
        if c in SPACE:
            index +=1
            break
        elif c in BRACKETS:
            break
        elif c in BRACKETS_END:
            break
        else:
            data += c
            index+=1
    return data, index

def parseSexp(content, index = 0):
    res = []
    c_len = len(content)
    data = ""
    bracket_end = ""
    while index < c_len:
        c = content[index]
        if c in BRACKETS:
            bracket_end = BRACKETS[c]
            data, index = parseSexp(content, index+1)
            res.append(data)
        elif c in QUOTE:
            data, index = quoteData(content, index)
            res.append(data)
        elif c in SPACE:
            index+=1;
        elif c in BRACKETS_END:
            index+=1
            break
        else:
            data, index = normalData(content, index)
            res.append(data)
    return res, index

def loadKicadNet(filename):
    file = io.open(filename, "r", encoding="utf-8")
    data = file.read()
    res = parseSexp(data)
    return res[0][0]


"""Tests for pytype_runner.py."""

import os
import unittest

from pytype import config as pytype_config
from pytype import file_utils
from pytype import module_utils
from pytype.tools.analyze_project import parse_args
from pytype.tools.analyze_project import pytype_runner

class TestBase(unittest.TestCase):

  @classmethod
  def setUpClass(cls):
    cls.parser = parse_args.make_parser()

class TestGetRunCmd(TestBase):
  """Test PytypeRunner.get_pytype_args()."""

  def setUp(self):
    self.runner = pytype_runner.PytypeRunner(
        [], [], self.parser.config_from_defaults())

  def get_basic_options(self, report_errors=False):
    module = module_utils.Module('foo', 'bar.py', 'bar')
    args = self.runner.get_pytype_args(module, report_errors)
    return pytype_config.Options(args)

  def test_pythonpath(self):
    self.assertEqual(self.get_basic_options().pythonpath, [self.runner.pyi_dir])

  def test_python_version(self):
    self.assertEqual(
        self.get_basic_options().python_version,
        tuple(int(i) for i in self.runner.python_version.split('.')))

  def test_output(self):
    self.assertEqual(self.get_basic_options().output,
                     os.path.join(self.runner.pyi_dir, 'bar.pyi'))

  def test_quick(self):
    self.assertTrue(self.get_basic_options().quick)

  def test_module_name(self):
    self.assertEqual(self.get_basic_options().module_name, 'bar')

  def test_error_reporting(self):
    # Disable error reporting
    options = self.get_basic_options(report_errors=False)
    self.assertFalse(options.report_errors)
    self.assertFalse(options.analyze_annotated)
    # Enable error reporting
    options = self.get_basic_options(report_errors=True)
    self.assertTrue(options.report_errors)
    self.assertTrue(options.analyze_annotated)

class TestYieldSortedModules(TestBase):
  """Tests for PytypeRunner.yield_sorted_modules()."""

  def normalize(self, d):
    return file_utils.expand_path(d).rstrip(os.sep) + os.sep

  def assert_sorted_modules_equal(self, mod_gen, expected_list):
    for path, target, name, expected_report_errors in expected_list:
      try:
        module, actual_report_errors = next(mod_gen)
      except StopIteration:
        raise AssertionError('Not enough modules')
      self.assertEqual(module, module_utils.Module(path, target, name))
      self.assertEqual(actual_report_errors, expected_report_errors)
    try:
      next(mod_gen)
    except StopIteration:
      pass
    else:
      # Too many modules
      raise AssertionError('Too many modules')

  def test_source(self):
    conf = self.parser.config_from_defaults()
    d = self.normalize('foo/')
    conf.pythonpath = [d]
    f = os.path.join(d, 'bar.py')
    runner = pytype_runner.PytypeRunner([f], [[f]], conf)
    self.assert_sorted_modules_equal(runner.yield_sorted_modules(),
                                     [(d, 'bar.py', 'bar', True)])

  def test_source_and_dep(self):
    conf = self.parser.config_from_defaults()
    d = self.normalize('foo/')
    conf.pythonpath = [d]
    source = os.path.join(d, 'bar.py')
    dep = os.path.join(d, 'baz.py')
    runner = pytype_runner.PytypeRunner([source], [[dep], [source]], conf)
    self.assert_sorted_modules_equal(
        runner.yield_sorted_modules(),
        [(d, 'baz.py', 'baz', False), (d, 'bar.py', 'bar', True)])

  def test_cycle(self):
    conf = self.parser.config_from_defaults()
    d = self.normalize('foo/')
    conf.pythonpath = [d]
    source = os.path.join(d, 'bar.py')
    dep = os.path.join(d, 'baz.py')
    runner = pytype_runner.PytypeRunner([source], [[dep, source]], conf)
    self.assert_sorted_modules_equal(
        runner.yield_sorted_modules(),
        [(d, 'baz.py', 'baz', False), (d, 'bar.py', 'bar', False),
         (d, 'baz.py', 'baz', False), (d, 'bar.py', 'bar', True)])

  def test_non_py_dep(self):
    conf = self.parser.config_from_defaults()
    d = self.normalize('foo/')
    conf.pythonpath = [d]
    dep = os.path.join(d, 'bar.so')
    runner = pytype_runner.PytypeRunner([], [[dep]], conf)
    self.assert_sorted_modules_equal(runner.yield_sorted_modules(), [])

  def test_non_pythonpath_dep(self):
    conf = self.parser.config_from_defaults()
    d = self.normalize('foo/')
    conf.pythonpath = [d]
    dep = file_utils.expand_path('bar/baz.py')
    runner = pytype_runner.PytypeRunner([], [[dep]], conf)
    self.assert_sorted_modules_equal(runner.yield_sorted_modules(), [])

if __name__ == '__main__':
  unittest.main()


"""
SQLite was free, local, and didn't require me to setup anything.  Nice
way to quickly store the data and get going.
"""

import sqlite3
import csv

# wd = Working Directory, where is the base path (data should be in subdir here)
wd = 'ENTER PATH HERE'

def create_table(cur):
    '''create the membership table'''

    cur.execute('DROP TABLE IF EXISTS membership')

    query = ('CREATE TABLE membership ( '
                'SCHOOL_YEAR TEXT, '
                'LEVEL_CODE TEXT, '
                'DIV_NUM INTEGER, '
                'DIV_NAME TEXT, '
                'SCH_NUM INTEGER, '
                'SCH_NAME TEXT, '
                'GRADE_CODE TEXT, '
                'FEDERAL_RACE_CODE INTEGER, '
                'GENDER TEXT, '
                'DISABILITY_FLAG TEXT, '
                'LEP_FLAG TEXT, '
                'DISADVANTAGED_FLAG TEXT, '
                'FALL_MEMBERSHIP_CNT INTEGER)')

    cur.execute(query)

def load_data(con, cur):
    '''inserts the csv rows into the database '''

    # fall_membership files
    for i in range(2006, 2011):
        insert(wd + 'data/' + str(i) + '_fall_membership.csv', cur)
        con.commit()

    # fm_divsch files
    for i in range(2011, 2014):
        insert(wd + 'data/' + str(i) + '_fm_divsch_1_69.csv', cur)
        con.commit()

        insert(wd + 'data/' + str(i) + '_fm_divsch_70_207.csv', cur)
        con.commit()

    # fm_state files
    for i in range(2011, 2014):
        insert(wd + 'data/' + str(i) + '_fm_state_div.csv', cur)
        con.commit()

def insert(file, cur):
    '''wrapper for the actual insert into the database'''

    with open(file, 'r') as f:
        reader = csv.reader(f)
        columns = next(reader)

        if(columns[0] == 'BEG_SCH_YR'):
            columns[0] = 'SCHOOL_YEAR'

        query = 'INSERT INTO membership ' + '({0}) values ({1})'
        query = query.format(','.join(columns), ','.join('?' * len(columns)))

        # fix the school names
        # Bailey's Crossroads Elementary == Bailey's Crossroads Elementary School
        for data in reader:
            # if(data[5][:-6] == 'School'):
            #     data[5] = data[5][:-7]

            cur.execute(query, data)

def check_data(cur):
    '''run query to print out counts for years for double checking'''

    query = 'select SCHOOL_YEAR, count(FALL_MEMBERSHIP_CNT) AS FALL_MEMBERSHIP \
        from membership GROUP BY SCHOOL_YEAR'
    cur.execute(query)

    total_rows = 0

    for row in cur.fetchall():
        print(row)
        total_rows += row[1]

    print('Total: ' + str(total_rows))

con = sqlite3.connect(wd + 'doe.db')
con.text_factory = str
cur = con.cursor()

create_table(cur)
load_data(con, cur)
check_data(cur)

con.commit()
con.close()


from rest_framework.generics import RetrieveAPIView, ListCreateAPIView, DestroyAPIView
from rest_framework.views import APIView
from rest_framework.renderers import JSONRenderer
from .serializers import CustomerAddressSerializer, CustomerSerializer, CustomerTableSerializer
from customers.models import Customer
from rest_framework.response import Response
from rest_framework.permissions import (
    IsAuthenticated,
    IsAdminUser,
)

class CustomerAddress(RetrieveAPIView):
    queryset = Customer.objects.all()
    serializer_class = CustomerAddressSerializer
    lookup_fields = [
        'address',
        'city',
        'state',
        'zip',
    ]

class CustomerAPIView(ListCreateAPIView):
    queryset = Customer.objects.all()
    serializer_class = CustomerSerializer

class CustomerTableView(ListCreateAPIView):
    queryset = Customer.objects.all()
    serializer_class = CustomerTableSerializer

class CustomerColumns(APIView):
    renderer_classes = (JSONRenderer,)

    def get(self, request, format=None):
        content = [
            {"name": "id", "title": "Customer ID", "breakpoints": "xs sm"},
            {"name": "full_name", "title": "Name"},
            {"name": "phone", "title": "Phone"},
            {"name": "email", "title": "Email", "breakpoints": "sm xs"},
            {"name": "actions", "title": "Actions", "type": "html", "sortable": False, "breakpoints": "sm xs"},
        ]
        return Response(content)

class CustomerDelete(DestroyAPIView):
    """
    Delete job
    """
    permission_classes = [
        IsAuthenticated,
        IsAdminUser,
    ]
    queryset = Customer.objects.all()


"""Handle transaction information interface."""

from marshmallow import fields

from ....messaging.models.base_record import (
    BaseExchangeRecord,
    BaseExchangeSchema,
)

from ....config.injection_context import InjectionContext

class TransactionRecord(BaseExchangeRecord):
    """Represents a single transaction record."""

    class Meta:
        """Transaction Record metadata."""

        schema_class = "TransactionRecordSchema"

    RECORD_ID_NAME = "transaction_id"
    CACHE_ENABLED = True
    TAG_NAMES = {"state", "thread_id", "connection_id", "transaction_type"}
    RECORD_TYPE = "transaction"
    STATE_INIT = "init"

    def __init__(
        self,
        *,
        transaction_id: str = None,
        _type: str = None,
        transaction_type:str = None,
        comment: str = None,
        signature_request: list = [],
        signature_response: list = [],
        timing: dict = {},
        formats: list = [],
        messages_attach: list = [],
        thread_id: str = None,
        connection_id: str = None,
        state: str = None,
        **kwargs,
    ):
        """Initialize a new TransactionRecord."""

        super().__init__(transaction_id, state or self.STATE_INIT, **kwargs)
        self._type = _type
        self.transaction_type = transaction_type
        self.comment = comment
        self.signature_request = signature_request
        self.signature_response = signature_response
        self.timing = timing
        self.formats = formats
        self.messages_attach = messages_attach
        self.thread_id = thread_id
        self.connection_id = connection_id

    @property
    def record_value(self) -> dict:
        """Accessor for the JSON record value generated for this credential exchange."""
        return {
            prop: getattr(self, prop)
            for prop in (
                "signature_request",
                "timing",
                "formats",
                "messages_attach",
                "signature_response",
                "transaction_type"
            )
        }

    @classmethod
    async def retrieve_by_connection_and_thread(
        cls, context: InjectionContext, connection_id: str, thread_id: str
    ) -> "TransactionRecord":
        """Retrieve a transaction record by connection and thread ID."""
        cache_key = f"credential_exchange_ctidx::{connection_id}::{thread_id}"
        record_id = await cls.get_cached_key(context, cache_key)
        if record_id:
            record = await cls.retrieve_by_id(context, record_id)
        else:
            record = await cls.retrieve_by_tag_filter(
                context,
                {"thread_id": thread_id},
                {"connection_id": connection_id} if connection_id else None,
            )
            await cls.set_cached_key(context, cache_key, record._id)
        return record

class TransactionRecordSchema(BaseExchangeSchema):
    """Schema to allow serialization/deserialization of transaction records."""

    class Meta:
        """TransactionRecordSchema metadata."""

        model_class = "TransactionRecord"

    _id = fields.Str(
        required=False, description="Connection identifier", example="any_example"
    )
    _type = fields.Str(
        required=False,
        description="Transaction type",
        example="The type of transaction",
    )
    transaction = fields.Str(
        required=False,
        description="The type of transaction (Schema or a Credential Definition"
    )
    signature_request = fields.List(
        fields.Dict(),
        required=False,
    )
    signature_response = fields.List(fields.Dict(), required=False)
    timing = fields.Dict(required=False)
    formats = fields.List(fields.Dict(), required=False)
    messages_attach = fields.List(fields.Dict(), required=False)
    thread_id = fields.Str(required=False, description="Thread Identifier")
    connection_id = fields.Str(
        required=False,
        description="The connection identifier for thie particular transaction record",
    )


import numpy as np
import matplotlib.pyplot as plt
import random
import os
from scipy.optimize import curve_fit
from scipy import special

def gaussian(x, mean, amplitude, standard_deviation):
    return amplitude * np.exp( - ((x - mean)**2 / (2*standard_deviation ** 2)))

def make_graphs(tracks, output_dir, title):
    vals, bin_edges = np.histogram(tracks, 100,(1,5))
    #print(vals)
    bin_middles = 0.5*(bin_edges[1:] + bin_edges[:-1])
    plt.errorbar(bin_middles, vals, np.sqrt(vals),  4.0/20.0,fmt='o',color='g')
    plt.title('Hist')
    plt.xlabel('randoms values')
    plt.savefig(f"{output_dir}/{title}.png")
    plt.clf()
def integral_approximation(f, a, b):
    return (b-a)*np.mean(f)

if __name__ == '__main__':

    output_dir = "hist_ana_20210510"
    os.makedirs(f'{output_dir}', exist_ok=True)

    list_cordinate_x = np.random.uniform(0,5,30000)
    list_cordinate_x.sort()
    print('list_cordinate_x',list_cordinate_x)
    print('len of list_cordinate_x is: ',len(list_cordinate_x))

    make_graphs(list_cordinate_x, output_dir, "hist_ran_numbers_01")
    nums = []
    mu = 0
    sigma = 0.2

    for i in range(30000):
        temp = random.gauss(mu, sigma)
        nums.append(temp)

    print('nums',nums)
    print('len of num: ',len(nums))
    list_cordinate_x_err = []
    for p,m in zip(list_cordinate_x,nums):
        list_cordinate_x_er = p + m
        list_cordinate_x_err.append(list_cordinate_x_er)
    print('list_cordinate_x_err',list_cordinate_x_err)
    print('len of list_cordinate_x_err: ',len(list_cordinate_x_err))
    make_graphs(list_cordinate_x_err, output_dir, "hist_ran_numbers_with_err_01")
    bin_heights, bin_edges = np.histogram(list_cordinate_x_err, 100,(0,5))
    #print(vals)
    bin_centers = 0.5*(bin_edges[1:] + bin_edges[:-1])
    plt.errorbar(bin_centers, bin_heights, np.sqrt(bin_heights),  4.0/20.0,fmt='o', ecolor='g', color='blue')

    popt, _ = curve_fit(gaussian, bin_centers, bin_heights, p0=[1., 1., 1.])
    print(popt)
    x_interval_for_fit = np.linspace(bin_edges[0], bin_edges[-1], 30000)
    plt.plot(x_interval_for_fit, gaussian(x_interval_for_fit, *popt), color='r', label='fit: mean=%5.3f, amp=%5.3f, std=%5.3f' % tuple(popt))
    plt.legend()
    plt.title('fitting')
    plt.xlim(-1,5.5)
    plt.xlabel('data_ran_num')
    plt.ylabel('Counts')
    plt.savefig(f"{output_dir}/fitting_ran_data.png")
    plt.show()
    plt.clf()

    plt.plot(x_interval_for_fit, gaussian(x_interval_for_fit, *popt), color='r', label='fit: mean=%5.3f, amp=%5.3f, std=%5.3f' % tuple(popt))
    plt.legend()
    plt.title('fitting')
    plt.xlim(-0.1,2)
    plt.xlabel('data_ran_num')
    plt.ylabel('Counts')
    plt.savefig(f"{output_dir}/fitting_ran_data_sxlim.png")
    plt.show()
    plt.clf()

    # plt.plot(list_cordinate_x_err, special.erf(list_cordinate_x_err))
    # plt.text(1.1,0,f"sigma = {np.std(special.erf(list_cordinate_x_err)):2.2f}", size=10)
    # print('special.erf(list_cordinate_x_err)',np.std(special.erf(list_cordinate_x_err)))
    # plt.xlabel('list_cordinate_vals')
    # plt.ylabel('$erf(x)$')
    # plt.xlim(-1,6)
    # plt.savefig(f"{output_dir}/error_function.png")
    # plt.show()
    # plt.clf()

    # mean = 0
    # standard_deviation = 1
    # a = 0
    # b = 1
    # fx = gaussian(x_interval_for_fit, mu, 30000, sigma)
    # approx = integral_approximation(fx,a,b)
    # print('approx',approx)


import pymongo
import scrapy
from scrapy import Request
from itemadapter import ItemAdapter

from pymongo import MongoClient
from pprint import pprint
from flask import Flask

class FlashbotSpider(scrapy.Spider):
    name = 'flashbot'
    allowed_domains = ['rss.jobsearch.monster.com']
    client = MongoClient()
    db = client.mydb
    my_collection = db.my_collection
    # Start the crawler at this URLs
    #start_urls = ['file://rss.jobsearch.monster.com/rssquery.ashx?q={query}']
    start_urls = ['http://rss.jobsearch.monster.com/rssquery.ashx?q={query}']

    thesaurus = ["machine learning", "machine", "learning", "big data", "big", "data"]

    LOG_LEVEL = "INFO"

    def parse(self, response):

        # We stat with this url
        url = self.start_urls[0]

        # Build and send a request for each word of the thesaurus
        for query in self.thesaurus:
            target = url.format(query=query)
            print("fetching the URL: %s" % target)
            if target.startswith("file://"):
                r = Request(target, callback=self.scrapit, dont_filter=True)
            else:
                r = Request(target, callback=self.scrapit)
            r.meta['query'] = query
            yield r

    def scrapit(self, response):
        query = response.meta["query"]

        # Base item with query used to this response
        print(query, response)

        # Scrap the data
        for doc in response.xpath("//item"):
            item = {"query": query}
            item["title"] = doc.xpath("title/text()").extract()
            item["description"] = doc.xpath("description/text()").extract()
            item["link"] = doc.xpath("link/text()").extract()
            item["pubDate"] = doc.xpath("pubDate/text()").extract()
            item["guid"] = doc.xpath("guid/text()").extract()

            guid = item ["guid"] [0]
            rest= self.my_collection.find({"guid":guid})
            if rest.count() ==0:
                self.my_collection.insert_one(item)
                print("document inserted", item ["title"])
                print("item scraped:", item["title"])
            yield item
         #to download data: scrapy crawl flashbot -o toto.xml

#flask:
app = Flask(__name__)
@app.route("/")
def root():
    return "Hello world"


""" run HypoInverse (main function)
  Usage:
    (1) modify template hyp control file to tune HypoInverse params
    (2) manually write velo mod (e.g., CRE file), include ref ele if necessary
    (3) set i/o paths in config file
    (4) python run_hyp.py
  Output:
    csv file: ot, lat, lon, dep, mag
    sum file (hyp)
    arc file (hyp)
"""
import os, glob
import numpy as np
import subprocess
import config

# i/o paths
cfg = config.Config()
net = cfg.net
ztr_rng = cfg.ztr_rng
ref_ele = cfg.ref_ele
fhyp_temp = cfg.fhyp_temp
fhyp = cfg.fhyp
f=open(fhyp_temp); lines=f.readlines(); f.close()
fsta = cfg.fsta_out
fpha = cfg.fpha_out
# if use CRE mod
pmod = cfg.pmod
smod = cfg.smod

# format input
os.system('python mk_sta.py')
os.system('python mk_phs.py')
for fname in glob.glob(cfg.fsums): os.unlink(fname)

# for all ztr
for ztri in ztr_rng:

    # 1. set control file
    out=open(fhyp,'w')
    for line in lines:
        if line[0:3]=='ZTR': line = "ZTR %s \n"%ztri
        if line[0:3]=='STA': line = "STA '%s' \n"%fsta
        if line[0:3]=='PHS': line = "PHS '%s' \n"%fpha
        if line[0:5]=='CRE 1': line = "CRE 1 '%s' %s T \n"%(pmod, ref_ele)
        if line[0:5]=='CRE 2': line = "CRE 2 '%s' %s T \n"%(smod, ref_ele)
        if line[0:3]=='PRT': line = "PRT 'output/%s_%s.ptr' \n"%(net, ztri)
        if line[0:3]=='ARC': line = "ARC 'output/%s_%s.arc' \n"%(net, ztri)
        if line[0:3]=='SUM': line = "SUM 'output/%s_%s.sum' \n"%(net, ztri)
        out.write(line)
    out.close()

    # 2. run hypoinverse
    p = subprocess.Popen(['hypoInv'], stdin=subprocess.PIPE)
    s = "@{}".format(fhyp)
    p.communicate(s.encode())

# format output
os.system('python sum2csv.py')
for fname in glob.glob('fort.*'): os.unlink(fname)


from unittest import mock

import pytest
from h_matchers import Any

from lms.models import GradingInfo, HGroup
from lms.resources import LTILaunchResource, OAuth2RedirectResource
from lms.resources._js_config import JSConfig
from lms.services import ConsumerKeyError, HAPIError

pytestmark = pytest.mark.usefixtures(
    "ai_getter",
    "grading_info_service",
    "grant_token_service",
    "h_api",
    "vitalsource_service",
)

class TestEnableContentItemSelectionMode:
    @pytest.mark.usefixtures("blackboard_files_enabled")
    def test_it(self, context, js_config):
        js_config.enable_content_item_selection_mode(
            mock.sentinel.form_action, mock.sentinel.form_fields
        )

        assert js_config.asdict()["mode"] == "content-item-selection"
        assert js_config.asdict()["filePicker"] == {
            "formAction": mock.sentinel.form_action,
            "formFields": mock.sentinel.form_fields,
            "google": {
                "clientId": "fake_client_id",
                "developerKey": "fake_developer_key",
                "origin": context.custom_canvas_api_domain,
            },
            "blackboard": {
                "enabled": True,
                "listFiles": {
                    "authUrl": "http://example.com/api/blackboard/oauth/authorize",
                    "path": "/api/blackboard/courses/test_course_id/files",
                },
            },
            "canvas": {
                "enabled": True,
                "ltiLaunchUrl": "http://example.com/lti_launches",
                "listFiles": {
                    "authUrl": "http://example.com/api/canvas/oauth/authorize",
                    "path": "/api/canvas/courses/test_course_id/files",
                },
            },
            "vitalSource": {
                "enabled": False,
            },
        }

    def test_google_picker_origin_falls_back_to_lms_url_if_theres_no_custom_canvas_api_domain(
        self, ai_getter, context, js_config
    ):
        context.custom_canvas_api_domain = None

        js_config.enable_content_item_selection_mode(
            mock.sentinel.form_action, mock.sentinel.form_fields
        )

        ai_getter.lms_url.assert_called_once_with()
        assert (
            js_config.asdict()["filePicker"]["google"]["origin"]
            == ai_getter.lms_url.return_value
        )

    def test_it_doesnt_enable_the_blackboard_file_picker_if_the_feature_flag_is_off(
        self, js_config
    ):
        js_config.enable_content_item_selection_mode(
            mock.sentinel.form_action, mock.sentinel.form_fields
        )

        assert not js_config.asdict()["filePicker"]["blackboard"]["enabled"]

    def test_it_doesnt_enable_the_canvas_file_picker_if_the_lms_isnt_Canvas(
        self, context, js_config
    ):
        context.is_canvas = False

        js_config.enable_content_item_selection_mode(
            mock.sentinel.form_action, mock.sentinel.form_fields
        )

        self.assert_canvas_file_picker_not_enabled(js_config)

    def test_it_doesnt_enable_the_canvas_file_picker_if_the_consumer_key_isnt_found_in_the_db(
        self, ai_getter, js_config
    ):
        ai_getter.developer_key.side_effect = ConsumerKeyError()

        js_config.enable_content_item_selection_mode(
            mock.sentinel.form_action, mock.sentinel.form_fields
        )

        self.assert_canvas_file_picker_not_enabled(js_config)

    def test_it_doesnt_enable_the_canvas_file_picker_if_we_dont_have_a_developer_key(
        self, ai_getter, js_config
    ):
        ai_getter.developer_key.return_value = None

        js_config.enable_content_item_selection_mode(
            mock.sentinel.form_action, mock.sentinel.form_fields
        )

        self.assert_canvas_file_picker_not_enabled(js_config)

    def test_it_doesnt_enable_the_canvas_file_picker_if_theres_no_custom_canvas_course_id(
        self, pyramid_request, js_config
    ):
        del pyramid_request.params["custom_canvas_course_id"]

        js_config.enable_content_item_selection_mode(
            mock.sentinel.form_action, mock.sentinel.form_fields
        )

        self.assert_canvas_file_picker_not_enabled(js_config)

    def test_it_enables_vitalsource_picker_if_feature_enabled(
        self, pyramid_request, js_config
    ):
        pyramid_request.feature = lambda feature: feature == "vitalsource"

        js_config.enable_content_item_selection_mode(
            mock.sentinel.form_action, mock.sentinel.form_fields
        )

        assert js_config.asdict()["filePicker"]["vitalSource"]["enabled"]

    def assert_canvas_file_picker_not_enabled(self, js_config):
        assert not js_config.asdict()["filePicker"]["canvas"]["enabled"]
        assert "courseId" not in js_config.asdict()

    @pytest.fixture
    def blackboard_files_enabled(self, pyramid_request):
        pyramid_request.feature = lambda feature: feature == "blackboard_files"

class TestAddCanvasFileID:
    """Unit tests for JSConfig.add_canvas_file_id()."""

    def test_it_adds_the_viaUrl_api_config(self, js_config):
        js_config.add_canvas_file_id(
            "example_canvas_course_id", "example_canvas_file_id"
        )

        assert js_config.asdict()["api"]["viaUrl"] == {
            "authUrl": "http://example.com/api/canvas/oauth/authorize",
            "path": "/api/canvas/courses/example_canvas_course_id/files/example_canvas_file_id/via_url",
        }

    def test_it_sets_the_canvas_file_id(self, js_config, submission_params):
        js_config.add_canvas_file_id(
            "example_canvas_course_id", "example_canvas_file_id"
        )

        assert submission_params()["canvas_file_id"] == "example_canvas_file_id"

class TestAddDocumentURL:
    """Unit tests for JSConfig.add_document_url()."""

    def test_it_adds_the_via_url(self, js_config, pyramid_request, via_url):
        js_config.add_document_url("example_document_url")

        via_url.assert_called_once_with(pyramid_request, "example_document_url")
        assert js_config.asdict()["viaUrl"] == via_url.return_value

    def test_it_adds_the_viaUrl_api_config_for_Blackboard_documents(self, js_config):
        js_config.add_document_url("blackboard://content-resource/xyz123")

        assert js_config.asdict()["api"]["viaUrl"] == {
            "authUrl": "http://example.com/api/blackboard/oauth/authorize",
            "path": "/api/blackboard/courses/test_course_id/files/xyz123/via_url",
        }

    def test_it_sets_the_document_url(self, js_config, submission_params):
        js_config.add_document_url("example_document_url")

        assert submission_params()["document_url"] == "example_document_url"

class TestAddVitalsourceLaunchConfig:
    """Unit tests for JSConfig.add_vitalsource_launch_config()."""

    def test_it_sets_vitalsource_config(
        self, js_config, pyramid_request, vitalsource_service
    ):
        js_config.add_vitalsource_launch_config("book-id", "/abc")

        vitalsource_service.get_launch_params.assert_called_with(
            "book-id", "/abc", pyramid_request.lti_user
        )
        assert js_config.asdict()["vitalSource"] == {
            "launchUrl": mock.sentinel.launch_url,
            "launchParams": mock.sentinel.launch_params,
        }

    def test_it_sets_submission_params(self, js_config, submission_params):
        js_config.add_vitalsource_launch_config("book-id", "/abc")

        assert submission_params() == Any.dict.containing(
            {
                "vitalsource_book_id": "book-id",
                "vitalsource_cfi": "/abc",
            }
        )

    @pytest.fixture
    def vitalsource_service(self, vitalsource_service):
        vitalsource_service.get_launch_params.return_value = (
            mock.sentinel.launch_url,
            mock.sentinel.launch_params,
        )
        return vitalsource_service

class TestAddCanvasFileIDAddDocumentURLCommon:
    """Tests common to both add_canvas_file_id() and add_document_url()."""

    def test_it_sets_the_canvas_submission_params(
        self, pyramid_request, method_caller, submission_params
    ):
        method_caller()

        assert (
            submission_params()["h_username"]
            == pyramid_request.lti_user.h_user.username
        )
        assert (
            submission_params()["lis_outcome_service_url"]
            == "example_lis_outcome_service_url"
        )
        assert (
            submission_params()["lis_result_sourcedid"]
            == "example_lis_result_sourcedid"
        )
        assert submission_params()["learner_canvas_user_id"] == "test_user_id"

    def test_it_doesnt_set_the_speedGrader_settings_if_the_LMS_isnt_Canvas(
        self, context, method_caller, js_config
    ):
        context.is_canvas = False

        method_caller()

        assert "speedGrader" not in js_config.asdict()["canvas"]

    def test_it_doesnt_set_the_speedGrader_settings_if_theres_no_lis_result_sourcedid(
        self, method_caller, js_config, pyramid_request
    ):
        del pyramid_request.params["lis_result_sourcedid"]

        method_caller()

        assert "speedGrader" not in js_config.asdict()["canvas"]

    def test_it_doesnt_set_the_speedGrader_settings_if_theres_no_lis_outcome_service_url(
        self, method_caller, js_config, pyramid_request
    ):
        del pyramid_request.params["lis_outcome_service_url"]

        method_caller()

        assert "speedGrader" not in js_config.asdict()["canvas"]

    @pytest.fixture(
        params=[
            {
                "method": "add_canvas_file_id",
                "args": ["example_canvas_course_id", "example_canvas_file_id"],
            },
            {"method": "add_document_url", "args": ["example_document_url"]},
        ]
    )
    def method_caller(self, js_config, request):
        """Return a function that calls the method-under-test with default args."""

        def method_caller():
            return getattr(js_config, request.param["method"])(*request.param["args"])

        return method_caller

class TestMaybeEnableGrading:
    def test_it_adds_the_grading_settings(
        self, js_config, grading_info_service, pyramid_request
    ):
        js_config.maybe_enable_grading()

        grading_info_service.get_by_assignment.assert_called_once_with(
            context_id="test_course_id",
            oauth_consumer_key=pyramid_request.lti_user.oauth_consumer_key,
            resource_link_id="TEST_RESOURCE_LINK_ID",
        )
        assert js_config.asdict()["grading"] == {
            "enabled": True,
            "assignmentName": "test_assignment_name",
            "courseName": "test_course_name",
            "students": [
                {
                    "LISOutcomeServiceUrl": f"test_lis_outcomes_service_url_{i}",
                    "LISResultSourcedId": f"test_lis_result_sourcedid_{i}",
                    "displayName": f"test_h_display_name_{i}",
                    "userid": f"acct:test_h_username_{i}@TEST_AUTHORITY",
                }
                for i in range(3)
            ],
        }

    @pytest.mark.usefixtures("user_is_learner")
    def test_it_does_nothing_if_the_user_isnt_an_instructor(self, js_config):
        js_config.maybe_enable_grading()

        assert not js_config.asdict().get("grading")

    def test_it_does_nothing_if_theres_no_lis_outcome_service_url(
        self, js_config, pyramid_request
    ):
        del pyramid_request.params["lis_outcome_service_url"]

        js_config.maybe_enable_grading()

        assert not js_config.asdict().get("grading")

    def test_it_does_nothing_in_Canvas(self, context, js_config):
        context.is_canvas = True

        js_config.maybe_enable_grading()

        assert not js_config.asdict().get("grading")

    @pytest.fixture
    def grading_info_service(self, grading_info_service):
        grading_info_service.get_by_assignment.return_value = [
            mock.create_autospec(
                GradingInfo,
                instance=True,
                spec_set=True,
                lis_result_sourcedid=f"test_lis_result_sourcedid_{i}",
                lis_outcome_service_url=f"test_lis_outcomes_service_url_{i}",
                h_username=f"test_h_username_{i}",
                h_display_name=f"test_h_display_name_{i}",
            )
            for i in range(3)
        ]
        return grading_info_service

    @pytest.fixture
    def context(self, context):
        context.is_canvas = False
        return context

    @pytest.fixture
    def pyramid_request(self, pyramid_request):
        pyramid_request.lti_user = pyramid_request.lti_user._replace(roles="Instructor")
        pyramid_request.params["context_title"] = "test_course_name"
        pyramid_request.params["resource_link_title"] = "test_assignment_name"
        return pyramid_request

class TestMaybeSetFocusedUser:
    def test_it_does_nothing_if_theres_no_focused_user_param(
        self, js_config, pyramid_request
    ):
        del pyramid_request.params["focused_user"]

        js_config.maybe_set_focused_user()

        assert "focus" not in js_config.asdict()["hypothesisClient"]

    def test_it_sets_the_focused_user_if_theres_a_focused_user_param(
        self, h_api, js_config
    ):
        js_config.maybe_set_focused_user()

        # It gets the display name from the h API.
        h_api.get_user.assert_called_once_with("example_h_username")
        # It sets the focused user.
        assert js_config.asdict()["hypothesisClient"]["focus"] == {
            "user": {
                "username": "example_h_username",
                "displayName": h_api.get_user.return_value.display_name,
            },
        }

    def test_display_name_falls_back_to_a_default_value(self, h_api, js_config):
        h_api.get_user.side_effect = HAPIError()

        js_config.maybe_set_focused_user()

        assert (
            js_config.asdict()["hypothesisClient"]["focus"]["user"]["displayName"]
            == "(Couldn't fetch student name)"
        )

    @pytest.fixture
    def pyramid_request(self, pyramid_request):
        pyramid_request.params["focused_user"] = "example_h_username"
        return pyramid_request

class TestJSConfigAuthToken:
    """Unit tests for the "authToken" sub-dict of JSConfig."""

    def test_it(
        self, authToken, bearer_token_schema, BearerTokenSchema, pyramid_request
    ):
        BearerTokenSchema.assert_called_once_with(pyramid_request)
        bearer_token_schema.authorization_param.assert_called_once_with(
            pyramid_request.lti_user
        )
        assert authToken == bearer_token_schema.authorization_param.return_value

    @pytest.fixture
    def authToken(self, config):
        return config["api"]["authToken"]

class TestJSConfigAPISync:
    """Unit tests for the api.sync sub-dict of JSConfig."""

    @pytest.mark.usefixtures("section_groups_on")
    def test_it(self, sync, pyramid_request, GroupInfo):
        assert sync == {
            "authUrl": "http://example.com/api/canvas/oauth/authorize",
            "path": "/api/canvas/sync",
            "data": {
                "course": {
                    "context_id": "test_context_id",
                    "custom_canvas_course_id": "test_custom_canvas_course_id",
                },
                "lms": {
                    "tool_consumer_instance_guid": "test_tool_consumer_instance_guid"
                },
                "group_info": {
                    key: value
                    for key, value in pyramid_request.params.items()
                    if key in GroupInfo.columns.return_value
                },
            },
        }

    @pytest.mark.usefixtures("section_groups_on", "learner_canvas_user_id")
    def test_it_adds_learner_canvas_user_id_for_SpeedGrader_launches(self, sync):
        assert sync["data"]["learner"] == {
            "canvas_user_id": "test_learner_canvas_user_id",
        }

    def test_its_None_if_section_groups_isnt_enabled(self, sync):
        assert sync is None

    @pytest.fixture
    def sync(self, config):
        return config["api"]["sync"]

    @pytest.fixture
    def learner_canvas_user_id(self, pyramid_request):
        pyramid_request.params["learner_canvas_user_id"] = "test_learner_canvas_user_id"

    @pytest.fixture
    def pyramid_request(self, pyramid_request):
        pyramid_request.params.clear()
        pyramid_request.params.update(
            {
                "context_id": "test_context_id",
                "custom_canvas_course_id": "test_custom_canvas_course_id",
                "tool_consumer_instance_guid": "test_tool_consumer_instance_guid",
                "foo": "bar",  # This item should be missing from group_info.
            }
        )
        return pyramid_request

class TestJSConfigDebug:
    """Unit tests for the "debug" sub-dict of JSConfig."""

    @pytest.mark.usefixtures("user_is_learner")
    def test_it_contains_debugging_info_about_the_users_role(self, config):
        assert "role:learner" in config["tags"]

    @pytest.fixture
    def config(self, config):
        return config["debug"]

class TestJSConfigHypothesisClient:
    """Unit tests for the "hypothesisClient" sub-dict of JSConfig."""

    def test_it_contains_one_service_config(self, config):
        assert len(config["services"]) == 1

    def test_it_includes_the_api_url(self, config):
        assert config["services"][0]["apiUrl"] == "https://example.com/api/"

    def test_it_includes_the_authority(self, config):
        assert config["services"][0]["authority"] == "TEST_AUTHORITY"

    def test_it_disables_share_links(self, config):
        assert not config["services"][0]["enableShareLinks"]

    def test_it_includes_grant_token(
        self, config, pyramid_request, grant_token_service
    ):
        grant_token_service.generate_token.assert_called_with(
            pyramid_request.lti_user.h_user
        )
        grant_token = config["services"][0]["grantToken"]
        assert grant_token == grant_token_service.generate_token.return_value

    def test_it_includes_the_group(self, config, context):
        groups = config["services"][0]["groups"]

        assert groups == [context.h_group.groupid.return_value]

    @pytest.mark.usefixtures("section_groups_on")
    def test_it_configures_the_client_to_fetch_the_groups_over_RPC(self, config):
        assert config["services"][0]["groups"] == "$rpc:requestGroups"

    @pytest.mark.usefixtures("provisioning_disabled")
    def test_it_is_empty_if_provisioning_feature_is_disabled(self, config):
        assert config == {}

    def test_it_is_mutable(self, config):
        config.update({"a_key": "a_value"})

        assert config["a_key"] == "a_value"

    @pytest.fixture
    def config(self, config):
        return config["hypothesisClient"]

class TestJSConfigRPCServer:
    """Unit tests for the "rpcServer" sub-dict of JSConfig."""

    def test_it(self, config):
        assert config == {"allowedOrigins": ["http://localhost:5000"]}

    @pytest.fixture
    def config(self, config):
        return config["rpcServer"]

class TestEnableCanvasOauth2RedirectErrorMode:
    def test_scope_error(self, js_config):
        js_config.enable_canvas_oauth2_redirect_error_mode(
            auth_url=None,
            error_details="Technical error",
            is_scope_invalid=True,
            requested_scopes=["scope_a", "scope_b"],
        )

        config = js_config.asdict()

        assert config["mode"] == "canvas-oauth2-redirect-error"
        assert config["canvasOAuth2RedirectError"] == {
            "authUrl": None,
            "invalidScope": True,
            "errorDetails": "Technical error",
            "scopes": ["scope_a", "scope_b"],
        }

    def test_other_error(self, js_config):
        auth_url = "https://lms.hypothes.is/auth/url"
        js_config.enable_canvas_oauth2_redirect_error_mode(
            auth_url=auth_url, error_details="Some error"
        )

        config = js_config.asdict()

        assert config["mode"] == "canvas-oauth2-redirect-error"
        assert config["canvasOAuth2RedirectError"] == {
            "authUrl": auth_url,
            "invalidScope": False,
            "errorDetails": "Some error",
            "scopes": [],
        }

    @pytest.fixture
    def js_config(self, context, pyramid_request):
        return JSConfig(context, pyramid_request)

    @pytest.fixture
    def pyramid_request(self, pyramid_request):
        pyramid_request.lti_user = None
        return pyramid_request

    @pytest.fixture
    def context(self):
        return mock.create_autospec(
            OAuth2RedirectResource, spec_set=True, instance=True
        )

@pytest.fixture(autouse=True)
def BearerTokenSchema(patch):
    return patch("lms.resources._js_config.BearerTokenSchema")

@pytest.fixture
def bearer_token_schema(BearerTokenSchema):
    return BearerTokenSchema.return_value

@pytest.fixture
def js_config(context, pyramid_request):
    config = JSConfig(context, pyramid_request)
    config.enable_lti_launch_mode()
    return config

@pytest.fixture
def config(js_config):
    return js_config.asdict()

@pytest.fixture
def submission_params(config):
    return lambda: config["canvas"]["speedGrader"]["submissionParams"]

@pytest.fixture
def context():
    return mock.create_autospec(
        LTILaunchResource,
        spec_set=True,
        instance=True,
        h_group=mock.create_autospec(HGroup, instance=True, spec_set=True),
        is_canvas=True,
        canvas_sections_enabled=False,
    )

@pytest.fixture
def section_groups_on(context):
    context.canvas_sections_enabled = True

@pytest.fixture
def pyramid_request(pyramid_request):
    pyramid_request.params.update(
        {
            "lis_result_sourcedid": "example_lis_result_sourcedid",
            "lis_outcome_service_url": "example_lis_outcome_service_url",
            "context_id": "test_course_id",
            "custom_canvas_course_id": "test_course_id",
            "custom_canvas_user_id": "test_user_id",
        }
    )
    return pyramid_request

@pytest.fixture
def provisioning_disabled(ai_getter):
    ai_getter.provisioning_enabled.return_value = False

@pytest.fixture(autouse=True)
def via_url(patch):
    return patch("lms.resources._js_config.via_url")

@pytest.fixture(autouse=True)
def GroupInfo(patch):
    group_info_class = patch("lms.resources._js_config.GroupInfo")
    group_info_class.columns.return_value = ["context_id", "custom_canvas_course_id"]
    return group_info_class


import sys
sys.path.insert(1, "../../../")
import h2o
from tests import pyunit_utils
from h2o.estimators.coxph import H2OCoxProportionalHazardsEstimator

def coxph_smoke():
    heart = h2o.import_file(pyunit_utils.locate("smalldata/coxph_test/heart.csv"))
    coxph = H2OCoxProportionalHazardsEstimator(start_column="start", stop_column="stop")
    coxph.train(x="age", y="event", training_frame=heart)

    assert coxph.model_id != ""
    assert coxph.formula() == "Surv(start, stop, event) ~ age", \
        "Expected formula to be 'Surv(start, stop, event) ~ age' but it was " + coxph.formula()

    pred = coxph.predict(test_data=heart)
    assert len(pred) == len(heart)

if __name__ == "__main__":
    pyunit_utils.standalone_test(coxph_smoke)
else:
    coxph_smoke()


import gdal
from gdalconst import *
import numpy as np
from multiprocessing.pool import Pool, ThreadPool
from functools import partial

def getData(band, cols, rows):
    return band.ReadAsArray(0,0,cols,rows).astype(np.float32)

def calcNDVI(inFile, outFile, bandRed, bandNir):
    """Calculate the NDVI index for the input file and write a new output file

    :param inFile: raster image file to process
    :type inFile: str
    :value inFile: path and name of the raster image file

    :param outFile: raster image output file
    :type outFile: str
    :value outFile: path and name of the output raster image file

    :param bandRed: number of the red raster band
    :type bandRed: int

    :param bandNir: number of the near infrared raster band
    :type bandNir: int
    """

    driver = gdal.AllRegister()
    img = gdal.Open(inFile, GA_ReadOnly)
    cols, rows = img.RasterXSize, img.RasterYSize

    band_Red = img.GetRasterBand(bandRed)
    band_Nir = img.GetRasterBand(bandNir)

    bands_Red_Nir = [band_Red, band_Nir]

    p = Pool(4)
    mapfunc_getData = partial(getData, cols = cols, rows = rows)
    data_bands_Red_Nir = p.map(mapfunc_getData, bands_Red_Nir)

    data_NDVI = (data_bands_Red_Nir[1] - data_bands_Red_Nir[0]) / (data_bands_Red_Nir[1] + data_bands_Red_Nir[0])

    bands_Red_Nir = None
    data_bands_Red_Nir = None

    driverout = gdal.GetDriverByName("GTiff")
    dst_ds = driverout.Create(outFile,cols, rows, 1, band_Red.DataType)
    dst_ds.SetGeoTransform(img.GetGeoTransform())
    dst_ds.SetProjection(img.GetProjection())
    outBand = dst_ds.GetRasterBand(1)

    outBand.WriteArray(data_NDVI, 0, 0)
    outBand.SetNoDataValue(-9999)

    data_NDVI = None
    dst_ds = None
    outBand = None

def readRaster(bandNr, inFile, dataType):
    """ Return the data of a raster image file

    :param inFile: raster image file to process
    :type inFile: str
    :value inFile: path and name of the raster image file

    :param bandNr: number of the band of the raster image file
    :type bandNr: int

    :param dataType: defines the data type of the returned data
    :type dataType: int
    :values dataType:
        1 - np.float16 (Half precision float: sign bit, 5 bits exponent, 10 bits mantissa)
        2 - np.float32 (Single precision float: sign bit, 8 bits exponent, 23 bits mantissa),
        3 - np.float64 (Double precision float: sign bit, 11 bits exponent, 52 bits mantissa),
        4 - np.int8 (Byte (-128 to 127)),
        5 - np.int16 (Integer (-32768 to 32767)),
        6 - np.int32 (Integer (-2147483648 to 2147483647)),
        7 - np.int64 (Integer (-9223372036854775808 to 9223372036854775807)),
        else - np.float64 (Double precision float: sign bit, 11 bits exponent, 52 bits mantissa)

    :further readings: http://docs.scipy.org/doc/numpy-1.10.1/user/basics.types.html"""

    driver = gdal.AllRegister()
    img = gdal.Open(inFile, GA_ReadOnly)
    cols, rows = img.RasterXSize, img.RasterYSize
    band = img.GetRasterBand(bandNr)

    if dataType == 1:
        dataBand = band.ReadAsArray(0,0,cols,rows).astype(np.float16)
    elif dataType == 2:
        dataBand = band.ReadAsArray(0,0,cols,rows).astype(np.float32)
    elif dataType == 3:
        dataBand = band.ReadAsArray(0,0,cols,rows).astype(np.float64)
    elif dataType == 4:
        dataBand = band.ReadAsArray(0,0,cols,rows).astype(np.int8)
    elif dataType == 5:
        dataBand = band.ReadAsArray(0,0,cols,rows).astype(np.int16)
    elif dataType == 6:
        dataBand = band.ReadAsArray(0,0,cols,rows).astype(np.int32)
    elif dataType == 7:
        dataBand = band.ReadAsArray(0,0,cols,rows).astype(np.int64)
    else:
        dataBand = band.ReadAsArray(0,0,cols,rows).astype(np.float64)

    return dataBand


import numpy as np

import matplotlib.pyplot as plt
import keras, time, os
from keras.layers import (Dense, Dropout, Activation, Flatten, Conv2D,
                          AveragePooling2D, MaxPooling2D, BatchNormalization)
import sys
logistic = __import__('logistic')
circular = __import__('fc2_circular_reg')

if __name__ == "__main__":
    ttt = time.time()

    path_file    = 'data/'
    path_results = 'results/'

    (isPU_train, isPU_val, _,
     image_train, image_val, _,
     rwi_train, rwi_val, _,
     rew_train, rew_val, _,
     Rpt_train, Rpt_val, _,
     j0pt_train, j0pt_val, _) = logistic.load_data(path_file)

    y_train = isPU_train.astype(int)
    y_val   = isPU_val.astype(int)

    X_train = image_train
    X_val   = image_val

    seed = 7
    np.random.seed(seed)
    batch_size = 10000

    model = keras.models.Sequential()
    #numfil = 1000
    regds = 0#1e-8
    #regth = 1e-4
    #regthobj = circular.reg_grad_theta(eta_range=10, phi_range=10,
    #                                   num_filters=numfil,
    #                                   num_channels=3, regth=regth, regl2=regds)

    model.add(Conv2D(filters=50, kernel_size=2, strides=(2, 2),
                     padding='same', data_format='channels_first',
                     input_shape=(3, 10, 10),
                     kernel_regularizer=keras.regularizers.l2(regds)))
    model.add(Activation('relu'))
    model.add(Conv2D(filters=100, kernel_size=5, strides=(1, 1),
                     padding='valid', data_format='channels_first',
                     kernel_regularizer=keras.regularizers.l2(regds)))
    model.add(Activation('relu'))
    model.add(Flatten())
    model.add(Dense(1, kernel_regularizer=keras.regularizers.l2(regds)))
    model.add(Activation('sigmoid'))

    model.compile(loss="binary_crossentropy",
                  optimizer='adam', metrics=['accuracy'])

    #sanity check
    los, acc = logistic.pred(model, batch_size, X_val, y_val, rwi_val, .5, True)

    current_file_name = os.path.basename(sys.argv[0])
    # __file__ is different in python and ipython.. 4152963

    chkpnt_name = path_results + current_file_name + '_' + str(regds) + '_chk_temp'
    chkpnt_ojb = keras.callbacks.ModelCheckpoint(
                     chkpnt_name, monitor='val_loss', verbose=0,
                     save_best_only=True, save_weights_only=False,
                     mode='auto', period=1)
    early_stop = keras.callbacks.EarlyStopping(
                     monitor='val_loss', min_delta=0, patience=50, verbose=1,
                     mode='auto')
    reduce_lr = keras.callbacks.ReduceLROnPlateau(
                     monitor='val_loss', factor=0.2, patience=10, verbose=1,
                     mode='auto', epsilon=1e-9)

    history = model.fit(X_train, y_train, epochs=1000, batch_size=batch_size,
                        verbose=1, sample_weight=rwi_train,
                        validation_data=(X_val, y_val, rwi_val),
                        callbacks=[chkpnt_ojb, early_stop, reduce_lr])

    #keras.regularizers.reg_grad_theta = circular.reg_grad_theta
    model = keras.models.load_model(chkpnt_name)

    los, acc = logistic.pred(model, batch_size, X_val, y_val, rwi_val, .5, True)

    output_name = (current_file_name + '_' + str(regds) + '_' +
                   time.strftime("%Y_%m_%d_%H_%M") + '_' + str(acc))

    model.save(path_results+output_name+'_mod')
    logistic.save_obj(history.history, path_results, output_name+'_his')

    #os.rename(chkpnt_name, path_results + output_name+'_his')
    os.remove(chkpnt_name)

    filters0 = model.layers[0].get_weights()[0]
    logistic.plot_filters(filters0, path_results, output_name+'_0', 10)
    filters2 = model.layers[2].get_weights()[0]
    logistic.plot_filters(filters2, path_results, output_name+'_2', 10)
    logistic.plot_history_loss(history.history, False, path_results, output_name)
    logistic.plot_model(model, path_results, output_name)

    plt.show()

    pred_val = np.squeeze(model.predict(X_val, batch_size=batch_size, verbose=1))
    effPU = logistic.roc(rew_val, isPU_val, True,  pred_val)
    effHS = logistic.roc(rew_val, isPU_val, False, pred_val)
    np.save(path_results + output_name + '_effPU', effPU)
    np.save(path_results + output_name + '_effHS', effHS)
    print(time.time()-ttt)


##############################################################################
#
# OSIS stands for Open Student Information System. It's an application
#    designed to manage the core business of higher education institutions,
#    such as universities, faculties, institutes and professional schools.
#    The core business involves the administration of students, teachers,
#    courses, programs and so on.
#
#    Copyright (C) 2015-2016 Université catholique de Louvain (http://www.uclouvain.be)
#
#    This program is free software: you can redistribute it and/or modify
#    it under the terms of the GNU General Public License as published by
#    the Free Software Foundation, either version 3 of the License, or
#    (at your option) any later version.
#
#    This program is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#    GNU General Public License for more details.
#
#    A copy of this license - GNU General Public License - is available
#    at the root of the source code of this program.  If not,
#    see http://www.gnu.org/licenses/.
#
##############################################################################
from django.db import models
from django.db.models import Q
from django.utils import timezone

from osis_common.models.serializable_model import SerializableModel, SerializableModelAdmin

class PropositionOfferAdmin(SerializableModelAdmin):
    list_display = ('proposition_dissertation', 'offer_proposition')
    raw_id_fields = ('proposition_dissertation', 'offer_proposition')
    search_fields = ('uuid', 'proposition_dissertation__title', 'offer_proposition__acronym',
                     'proposition_dissertation__author__person__last_name',
                     'proposition_dissertation__author__person__first_name')

class PropositionOffer(SerializableModel):
    proposition_dissertation = models.ForeignKey('PropositionDissertation', on_delete=models.CASCADE)
    offer_proposition = models.ForeignKey('OfferProposition', on_delete=models.CASCADE)

    def __str__(self):
        return str(self.offer_proposition)

def find_visible_by_education_groups(education_groups):
    now = timezone.now()
    return PropositionOffer.objects.filter(
        proposition_dissertation__active=True,
        proposition_dissertation__visibility=True,
        offer_proposition__education_group__in=education_groups,
        offer_proposition__start_visibility_proposition__lte=now,
        offer_proposition__end_visibility_proposition__gte=now
    )

def search(education_groups, terms, active=None, visibility=None):
    queryset = PropositionOffer.objects.filter(proposition_dissertation__active=True,
                                               proposition_dissertation__visibility=True,
                                               offer_proposition__education_group__in=education_groups,
                                               offer_proposition__start_visibility_proposition__lte=timezone.now()) \
        .distinct()
    if terms:
        queryset = queryset.filter(
            Q(proposition_dissertation__title__icontains=terms) |
            Q(proposition_dissertation__description__icontains=terms) |
            Q(proposition_dissertation__author__person__first_name__icontains=terms) |
            Q(proposition_dissertation__author__person__middle_name__icontains=terms) |
            Q(proposition_dissertation__author__person__last_name__icontains=terms) |
            Q(offer_proposition__education_group__educationgroupyear__acronym__icontains=terms)
        )
    if active:
        queryset = queryset.filter(proposition_dissertation__active=active)
    elif visibility:
        queryset = queryset.filter(proposition_dissertation__visibility=visibility)
    queryset = queryset.distinct()
    return queryset


import cons
import minas

def crear_matriz():
    "Crea una matriz vacía"

    M = [[cons.VACIO for _ in range(cons.DIM_HORIZONTAL+1)]for _ in range(cons.DIM_VERTICAL+1)]
    list(cons.ABC)
    M[0][0] = " "
    for i in range(1,cons.DIM_HORIZONTAL+1):
        M[0][i] = i
    for i in range (1,cons.DIM_VERTICAL+1):
        M[i][0] = cons.ABC[i-1]
    return M

#M = crear_matriz()
#Hasta acá: Matriz Blanqueada.
#Márgenes superior e izquierdo numerado (num y letras respectivamente).

def imprimir_tablero(M):
    "imprime la matriz en forma de tablero"

    for i in range(cons.DIM_VERTICAL+1):
        lista = []
        for j in range(cons.DIM_HORIZONTAL+1):
            lista.append(str(M[i][j]))
        print(" | ".join(lista))

def es_matriz_ganadora(M,mapa_minas):
    """indica si todas las ubicaciones de la matriz
    han sido descubiertas exceptuando aquellas que contienen mina"""

    for i in range(1,cons.DIM_VERTICAL+1):
        for j in range(1,cons.DIM_HORIZONTAL+1):
            if M[i][j] == cons.VACIO and not (cons.ABC[i-1],j) in mapa_minas:
                return False
            elif M[i][j] == cons.FLAG and (cons.ABC[i-1],j) in mapa_minas:
                continue
    return True


#!/usr/bin/env python
import numpy as np
import matplotlib
matplotlib.use('Agg') # force matplotlib to NOT use X (https://stackoverflow.com/questions/2801882/generating-a-png-with-matplotlib-when-display-is-undefined)
import matplotlib.pyplot as plt

LOGS_DIR = '/home/cs244/logs/'
PLOTS_DIR = '/home/cs244/plots/'

NNODES = 350

upload_caps = [16, 64, 128, 300, 512, 1024]
upload_caps_strs = [str(i) for i in upload_caps]
regular_durations = []
bittyrant_durations = []

# Generate an upload cap distribution that roughly
# models the CDF in Figure 1 of the Anderson paper
# (values are in KBps)
def generate_distribution(i):
    if 0 <= i <= 100:
        return 64
    if 101 <= i <= 200:
        return 96
    if 201 <= i <= 300:
        return 128
    if 301 <= i <= 340:
        return 1000
    else:
        return 10000

# Generate the upload cap distribution used by the peer nodes
upload_x = range(0, NNODES)
upload_y = [generate_distribution(x) for x in upload_x]

print("generating and serving plots from the log files...")

# we want to get two arrays of durations, for BitTyrant and regular,
# from the start and end times in the logs

with open(LOGS_DIR + 'regular', 'r') as ins:
    for line in ins:
        toks = line.split()
        regular_durations.append(int(toks[1]) - int(toks[0]))

with open(LOGS_DIR + 'bittyrant', 'r') as ins:
    for line in ins:
        toks = line.split()
        bittyrant_durations.append(int(toks[1]) - int(toks[0]))

# Now, generate the plots
fig, ax = plt.subplots()
index = np.arange(len(regular_durations))
bar_width = 0.35
opacity = 0.8

rects1 = plt.bar(index, regular_durations, bar_width, alpha=opacity, color='b', label='non-BitTyrant')
rects2 = plt.bar(index + bar_width, bittyrant_durations, bar_width, alpha=opacity, color='r', label='BitTyrant')

plt.xlabel('Peer upload cap')
plt.ylabel('Time-to-completion (seconds)')
plt.title('Time-to-completion vs. allowed upload cap')
plt.xticks(index + bar_width, upload_caps_strs)
plt.legend()

# TODO
#os.system('sudo python -m SimpleHTTPServer 80')

plt.tight_layout()
fig.savefig(PLOTS_DIR + 'fig1.png')
plt.close(fig)

# and generate the distribution plot
ax = plt.axes()
ax.set_yscale('log')

plt.bar(upload_x, upload_y, align="center")

plt.title("Swarm peer upload cap vs. peer number")
plt.ylabel("Upload cap (KBps)")
plt.grid(True)

plt.tight_layout()

fig = plt.figure(1)
fig.savefig(PLOTS_DIR + 'fig2.png')
plt.close(fig)

#plt.show()

# TODO
#os.system('sudo python -m SimpleHTTPServer 80')

for dur in regular_durations:
    print("reg dur: " + str(dur))

print("-----")

for dur in bittyrant_durations:
    print("bittyrant dur: " + str(dur))


from __future__ import print_function
from server.api.classifier.common.mva19 import Estimator, preprocess
import numpy as np
import cv2
import time
import argparse
import tensorflow as tf
from server.api.classifier.utils import detector_utils as detector_utils
from server.api.classifier.utils import recognizer_utils as recognizer_utils
import os
import numpy as np
from keras.models import *
import time
import shutil

os.environ['TF_CPP_MIN_LOG_LEVEL'] = '2'
map_characters = {0: 'A', 1: 'B', 2: 'C', 3: 'D', 4: 'E', 5: 'F', 6: 'G', 7: 'H', 8: 'I', 9: 'J', 10: 'K', 11: 'L', 12: 'M', 13: 'N', 14: 'O', 15: 'P', 16: 'Q', 17: 'R', 18: 'S', 19: 'T', 20: 'U', 21: 'V', 22: 'W', 23: 'X', 24: 'Y', 25: 'Z'}
# map_characters = {0: 'A', 1: 'B', 2: 'C', 3: 'D', 4: 'E', 5: 'F', 6: 'G', 7: 'H', 8: 'I', 9: 'L'}

generating_dataset = False

refersh_data = True

letter = 'I'
idx = 0

if generating_dataset:
    if refersh_data:
        if os.path.isdir('cust_data/{}'.format(letter)):
            shutil.rmtree('cust_data/{}'.format(letter))
        os.mkdir('cust_data/{}'.format(letter))

def get_bbox(img_src):
    parser = argparse.ArgumentParser()
    parser.add_argument(
        '-nhands',
        '--num_hands',
        dest='num_hands',
        type=int,
        default=1,
        help='Max number of hands to detect.')
    args = parser.parse_args()
    if img_src is None:
        print('Load Fail')
    else:
        boxes, scores = detector_utils.detect_objects(
            img_src, detection_graph, sess)
        boxes_to_recog, scores_to_show = detector_utils.draw_box_on_image(
            args.num_hands, score_thresh, scores, boxes,
            img_src.shape[1], img_src.shape[0], img_src)
        boxes_roi = boxes_to_recog
        if len(boxes_roi) > 0:
            return True
        else:
            return False

if __name__ == "__main__":

    model_file = "./models/mobnet4f_cmu_adadelta_t1_model.pb"
    input_layer = "input_1"
    output_layer = "k2tfout_0"

    detection_graph, sess = detector_utils.load_inference_graph()
    sess = tf.Session(graph=detection_graph)
    score_thresh = 0.1

    stride = 4
    boxsize = 224

    estimator = Estimator(model_file, input_layer, output_layer)

    # json_file = open('rec_model.json', 'r')
    json_file = open('rec_model.json', 'r')
    rec_model_json = json_file.read()
    json_file.close()
    rec_model = model_from_json(rec_model_json)
    # rec_model.load_weights("rec_model_4epochs.h5")
    rec_model.load_weights("rec_model_17epochs.h5")
    print("Loaded rec model from disk")

    cap = cv2.VideoCapture(0)
    cap.set(cv2.CAP_PROP_BRIGHTNESS, 0.4)

    paused = True
    delay = {False: 1, True: 0}

    k = 0
    while k != ord('q'):
        # ret, frame = cap.read()
        # if not ret:
        #     raise Exception("VideoCapture.read() returned False")
        frame = cv2.imread('img path')
        if frame is None:
            continue
        tic = time.time()
        if get_bbox(frame):

            crop_res = cv2.resize(frame, (boxsize, boxsize))
            img, pad = preprocess(crop_res, boxsize, stride)

            tic = time.time()
            hm = estimator.predict(img)
            dt = time.time() - tic

            hm = cv2.resize(hm, (0, 0), fx=stride, fy=stride)
            bg = cv2.normalize(hm[:, :, -1], None, 0, 255, cv2.NORM_MINMAX, cv2.CV_8UC1)
            viz = cv2.normalize(np.sum(hm[:, :, :-1], axis=2), None, 0, 255, cv2.NORM_MINMAX, cv2.CV_8UC1)

            if generating_dataset:
                cv2.imwrite('cust_data/{}/{}.jpg'.format(letter, idx), bg)
                print(idx)

            cv2.imwrite('msk/test/{}.jpg'.format(idx), bg)
            cv2.imwrite('ori/test/{}.jpg'.format(idx), crop_res)

            im = cv2.cvtColor(bg, cv2.COLOR_GRAY2BGR)
            im = cv2.resize(im, (100, 100))
            im = np.expand_dims(im, axis=0)
            result = rec_model.predict(im)
            # Gives the result
            result_letter = map_characters[np.argmax(result[0])]
            frame = cv2.putText(frame, str(result_letter) + ' FPS:{}'.format(int(1/dt)), (50, 50), cv2.FONT_HERSHEY_COMPLEX, 0.5, (0, 0, 255), 1)

            idx += 1
        else:
            bg = np.zeros((224, 224), np.int8)
            viz = np.zeros((224, 224), np.int8)
            viz.fill(255)
            dt = time.time() - tic
            frame = cv2.putText(frame, 'MOVE HAND TO CENTER PLEASE FPS:{}'.format(int(1/dt)), (50, 50), cv2.FONT_HERSHEY_COMPLEX, 0.5, (255, 255, 0), 1)

        cv2.imshow("Background", bg)
        cv2.imshow("All joint heatmaps", viz)
        cv2.imshow("Input frame", frame)

        k = cv2.waitKey(delay[paused])

        if k & 0xFF == ord('p'):
            paused = not paused
        elif k & 0xFF == ord('q'):
            break
    cap.release()
    cv2.destroyAllWindows()


#Scraping Numbers from HTML using BeautifulSoup In this assignment you will
#write a Python program similar to http://www.py4e.com/code3/urllink2.py.
#The program will use urllib to read the HTML from the data files below,
#and parse the data, extracting numbers and compute the sum of the numbers in
#the file.
#We provide two files for this assignment. One is a sample file where we give
#you the sum for your testing and the other is the actual data you need to
#process for the assignment.
#Sample data: http://py4e-data.dr-chuck.net/comments_42.html (Sum=2553)
#Actual data: http://py4e-data.dr-chuck.net/comments_695758.html (Sum ends with 53)
#You do not need to save these files to your folder since your program will read
#the data directly from the URL. Note: Each student will have a distinct data
#url for the assignment - so only use your own data url for analysis.
#You are to find all the <span> tags in the file and pull out the numbers from
#the tag and sum the numbers.
#Look at the sample code provided. It shows how to find all of a certain kind of
#tag, loop through the tags and extract the various aspects of the tags.

import urllib.request, urllib.parse, urllib.error
from bs4 import BeautifulSoup

my_url = input('Enter - ')

my_html = urllib.request.urlopen(my_url).read()
my_soup = BeautifulSoup(my_html,'html.parser')
tag = my_soup('span')
count=0
sum=0
for i in tag:
	x=int(i.text)
	count+=1
	sum = sum + x
print(count)
print(sum)