#!/usr/bin/env python
# -*- coding: utf-8 -*-
import threading

import requests
import urllib
import os
__author__ = 'wangjj'
__mtime__ = '20160929下午 3:19'

gImageList = []
gCondition = threading.Condition()

class Producer(threading.Thread):

    def run(self):
        global gImageList
        global gCondition
        print ('%s: started' % threading.current_thread())
        imgs = download_wallpaper()
        gCondition.acquire()
        for i in imgs:
            if 'downloadUrl' in i:
                gImageList.append(i['downloadUrl'])
        print ('%s: produce finished. Left :%s' %
               (threading.current_thread(), len(gImageList)))
        gCondition.notify_all()
        gCondition.release()

class Consumer(threading.Thread):

    def run(self):
        global gImageList
        global gCondition
        print ('%s: started' % threading.current_thread())
        while True:
            gCondition.acquire()
            print ('%s: trying to download from pool. pool size is %d' %
                   (threading.current_thread(), len(gImageList)))
            while len(gImageList) == 0:
                gCondition.wait()
                print (
                    '%s: wake up. poos size is %d' %
                    (threading.Thread(), len(gImageList)))
            url = gImageList.pop()
            gCondition.release()
            _download_image(url)

def download_wallpaper():
    # http://image.baidu.com/data/imgs?
    # pn=0&rn=18&col=%E5%A3%81%E7%BA%B8&
    # tag=%E5%9B%BD%E5%AE%B6%E5%9C%B0%E7%90%86&
    # tag3=&width=1920&height=1080&ic=0&ie=utf8&oe=utf-8&
    # image_id=&fr=channel&p=channel&from=1&
    # app=img.browse.channel.wallpaper&t=0.6772206311645761
    url = 'http://image.baidu.com/data/imgs'
    params = {
        'pn': 0,
        'rn': 26,
        'col': '壁纸',
        'tag': '国家地理',
        'tag3': '',
        'width': 1920,
        'height': 900,
        'ic': 0,
        'ie': 'utf8',
        'oe': 'utf-8',
        'image_id': '',
        'fr': 'channel',
        'p': 'channel',
        'from': 1,
        'app': 'img.browse.channel.wallpaper',
        't': '0.6772206311645761'
    }
    r = requests.get(url, params=params)
    # print(r.json())
    imgs = r.json()['imgs']
    print ('%s: totally %d imgs' % (threading.current_thread(), len(imgs)))
    # for i in imgs:
    #     if 'downloadUrl' in i:
    #         _download_image(i['downloadUrl'])
    return imgs

def _download_image(url, folder='image2'):
    if not os.path.isdir(folder):
        os.mkdir(folder)

    print ('downloading %s' % url)

    def _fname(s):
        return os.path.join(folder, os.path.split(s)[1])

    urllib.urlretrieve(url, _fname(url))

if __name__ == '__main__':
    Producer().start()

    for i in range(10):
        Consumer().start()


import sys
sys.path.append('../../')

from npcnn import read_onnx
from skimage import io
from matplotlib import pyplot as plt
import numpy as np
from time import time

img = io.imread('test.jpg')
img = img.astype(np.float32)
x = img[None, None, :, :]/255

net = read_onnx('unet')

y = net(x)
start = time()
for i in range(10):
    y = net(x)
print('unet npcnn time:', time()-start)

plt.subplot(121)
plt.title('gray')
plt.imshow(img, 'gray')
plt.subplot(122)
plt.title('segment')
plt.imshow(y[0, 0, :, :])
plt.show()


from math import log10, sqrt
import cv2
import numpy as np

def PSNR(before, after):
    mse = np.mean((before - after) ** 2)
    if(mse == 0): # MSE is zero means no noise is present in the signal .
        # Therefore PSNR have no importance.
        return 100
    max_pixel = 255.0
    psnr = 20 * log10(max_pixel / sqrt(mse))
    return psnr

before = cv2.imread("before.png")
after = cv2.imread("after.png", 1)
value = PSNR(before, after)
print(f"PSNR value is {value} dB")


# -*- coding:utf8 -*- #
"""
计算行为计数的统计信息并进行可视化
"""

import cPickle
import numpy as np
import matplotlib.pyplot as plt

# 时间参数
lookAhead = 3   # 前看时间长度

# 读取计数数据
cnt_view = cPickle.load(open("../../AliTianChi_data/count/cnt_view.pkl", 'rb'))
cnt_like = cPickle.load(open("../../AliTianChi_data/count/cnt_like.pkl", 'rb'))
cnt_addChart = cPickle.load(open("../../AliTianChi_data/count/cnt_addChart.pkl", 'rb'))
cnt_buy = cPickle.load(open("../../AliTianChi_data/count/cnt_buy.pkl", 'rb'))

# 分正负样本，并对一个月进行平均
mean_view_pos = np.reshape(np.mean(cnt_view[1, :, :], 0), lookAhead*24)
mean_like_pos = np.reshape(np.mean(cnt_like[1, :, :], 0), lookAhead*24)
mean_addChart_pos = np.reshape(np.mean(cnt_addChart[1, :, :], 0), lookAhead*24)
mean_buy_pos = np.reshape(np.mean(cnt_buy[1, :, :], 0), lookAhead*24)
mean_view_neg = np.reshape(np.mean(cnt_view[0, :, :], 0), lookAhead*24)
mean_like_neg = np.reshape(np.mean(cnt_like[0, :, :], 0), lookAhead*24)
mean_addChart_neg = np.reshape(np.mean(cnt_addChart[0, :, :], 0), lookAhead*24)
mean_buy_neg = np.reshape(np.mean(cnt_buy[0, :, :], 0), lookAhead*24)

# 进行绘图
t = np.linspace(1, lookAhead*24, lookAhead*24)
plt.plot(t, mean_buy_pos/np.max( mean_buy_pos), 'b-')
plt.plot(t, mean_buy_neg/np.max( mean_buy_neg), 'r-')
plt.show()


#!/usr/bin/env python
""" RNN + LSTM
"""
from convertData import TwoWayDict
from inputParser import get_parser
from collections import defaultdict
import sys
import gzip
import numpy as np
import theano
import theano.tensor as T

import logging
import time
import os
import datetime
import cPickle as pickle
import update_list

from utils import case_insensitive_import

logging.basicConfig(stream=sys.stdout, level=logging.DEBUG)
logger = logging.getLogger(__name__)
mode = theano.Mode(linker='cvm')

class LSTM(object):
	"""	   LSTM  class
	"""
	def __init__(self, input0, input1, input2, n_in, n_hidden, n_out, ylen0 = 1, ylen1 = 1, ylen2 = 2, activation=T.tanh,
				 output_type='real', use_symbolic_softmax=False, prange = 0.01, lstm_flag = False):

		self.lstm_flag = lstm_flag
		self.output_type = output_type

		self.input0 = input0
		self.input1 = input1
		self.input2 = input2

		self.activation = activation
		self.n_out = n_out
		self.n_hidden = n_hidden

		if lstm_flag:
			# hidden to hidden connections
			W_hh_init = np.asarray(np.random.uniform(size=(n_hidden, n_hidden),
												  low=-.01, high=.01),
												  dtype=theano.config.floatX)
			self.W_hh = theano.shared(value=W_hh_init, name='Whh', borrow = True)

			W_hz_init = np.asarray(np.random.uniform(size=(n_hidden, n_hidden),
												  low=-.01, high=.01),
												  dtype=theano.config.floatX)
			self.W_hz = theano.shared(value=W_hz_init, name='Whz', borrow = True)

			W_hr_init = np.asarray(np.random.uniform(size=(n_hidden, n_hidden),
												  low=-.01, high=.01),
												  dtype=theano.config.floatX)
			self.W_hr = theano.shared(value=W_hr_init, name='Whr', borrow = True)

			# input to hidden layer weights
			W_xh_init = np.asarray(np.random.uniform(size=(n_in, n_hidden),
													 low=-.01, high=.01),
													 dtype=theano.config.floatX)
			self.W_xh = theano.shared(value=W_xh_init, name='Wxh', borrow = True)

			W_xz_init = np.asarray(np.random.uniform(size=(n_in, n_hidden),
													 low=-.01, high=.01),
													 dtype=theano.config.floatX)
			self.W_xz = theano.shared(value=W_xz_init, name='Wxz', borrow = True)

			W_xr_init = np.asarray(np.random.uniform(size=(n_in, n_hidden),
													 low=-.01, high=.01),
													 dtype=theano.config.floatX)
			self.W_xr = theano.shared(value=W_xr_init, name='Wxr', borrow = True)

			h0_init = np.zeros((n_hidden,), dtype=theano.config.floatX)
			self.h0 = theano.shared(value=h0_init, name='h0', borrow = True)

			# biases
			bhh_init = np.zeros((n_hidden,), dtype=theano.config.floatX)
			self.bhh = theano.shared(value=bhh_init, name='bhh', borrow = True)

			bhz_init = np.zeros((n_hidden,), dtype=theano.config.floatX)
			self.bhz = theano.shared(value=bhz_init, name='bhz', borrow = True)

			bhr_init = np.zeros((n_hidden,), dtype=theano.config.floatX)
			self.bhr = theano.shared(value=bhr_init, name='bhr', borrow = True)

			bxh_init = np.zeros((n_hidden,), dtype=theano.config.floatX)
			self.bxh = theano.shared(value=bxh_init, name='bxh', borrow = True)

			bxz_init = np.zeros((n_hidden,), dtype=theano.config.floatX)
			self.bxz = theano.shared(value=bxz_init, name='bxz', borrow = True)

			bxr_init = np.zeros((n_hidden,), dtype=theano.config.floatX)
			self.bxr = theano.shared(value=bxr_init, name='bxr', borrow = True)

			# softmax layer
			W_log_init = np.asarray(np.random.uniform(size=( 3*n_hidden, n_out),
													  low=-.01, high=.01),
													  dtype=theano.config.floatX)
			self.W_log = theano.shared(value=W_log_init, name='W_log', borrow = True)

			blog_init = np.zeros((n_out,), dtype=theano.config.floatX)
			self.blog = theano.shared(value=blog_init, name='blog', borrow = True)

			# parameters
			self.params = [self.W_hh, self.W_hz, self.W_hr,  self.W_xh, self.W_xz, self.W_xr, self.W_log, self.h0, self.bhh , self.bhz, self.bhr, self.bxh, self.bxz, self.bxr, self.blog]

			def step(x_t, h_tm1):
				z = self.activation( T.dot(x_t, self.W_xz) + self.bxz + T.dot(h_tm1, self.W_hz) + self.bhz )
				r = self.activation( T.dot(x_t, self.W_xr) + self.bxr + T.dot(h_tm1, self.W_hr) + self.bhr)
				htilda = self.activation( T.dot(x_t, self.W_xh) + self.bxh + T.dot(r * h_tm1, self.W_hh) + self.bhh)
				h_t = z*h_tm1 + (1-z)*htilda
				return h_t

			self.s0h, _ = theano.scan(step, sequences=self.input0, outputs_info=[self.h0])
			self.s1h, _ = theano.scan(step, sequences=self.input1, outputs_info=[self.h0])
			self.s2h, _ = theano.scan(step, sequences=self.input2, outputs_info=[self.h0])

			self.L1 = abs(self.W_xh.sum()) + abs(self.W_xz.sum()) + abs(self.W_xr.sum()) + abs(self.W_hh.sum()) + abs(self.W_hz.sum()) + abs(self.W_hr.sum())
			self.L2_sqr = (self.W_xh ** 2).sum() + (self.W_xz ** 2).sum() + (self.W_xr ** 2).sum() + (self.W_hh ** 2).sum() + (self.W_hz ** 2).sum() + (self.W_hr ** 2).sum()

		else:
			W_init = np.asarray(np.random.uniform(size=(n_hidden, n_hidden),
												  low=-.01, high=.01),
												  dtype=theano.config.floatX)
			self.W = theano.shared(value=W_init, name='W', borrow = True)
			# input to hidden layer weights
			W_in_init = np.asarray(np.random.uniform(size=(n_in, n_hidden),
													 low=-.01, high=.01),
													 dtype=theano.config.floatX)
			self.W_in = theano.shared(value=W_in_init, name='W_in', borrow = True)

			h0_init = np.zeros((n_hidden,), dtype=theano.config.floatX)
			self.h0 = theano.shared(value=h0_init, name='h0', borrow = True)

			bh_init = np.zeros((n_hidden,), dtype=theano.config.floatX)
			self.bh = theano.shared(value=bh_init, name='bh', borrow = True)

			W_log_init = np.asarray(np.random.uniform(size=( 3*n_hidden, n_out),
													  low=-.01, high=.01),
													  dtype=theano.config.floatX)
			self.W_log = theano.shared(value=W_log_init, name='W_log', borrow = True)

			blog_init = np.zeros((n_out,), dtype=theano.config.floatX)
			self.blog = theano.shared(value=blog_init, name='blog', borrow = True)

			self.params = [self.W, self.W_in, self.W_log, self.h0, self.bh, self.blog]

			def step0(x_t, h_tm1):
				h_t = self.activation(T.dot(x_t, self.W_in) + T.dot(h_tm1, self.W) + self.bh)
				return h_t

			def step1(x_t, h_tm1):
				h_t = self.activation(T.dot(x_t, self.W_in) + T.dot(h_tm1, self.W) + self.bh)
				return h_t

			def step2(x_t, h_tm1):
				h_t = self.activation(T.dot(x_t, self.W_in) + T.dot(h_tm1, self.W) + self.bh)
				return h_t

			self.s0h, _ = theano.scan(step0, sequences=self.input0, outputs_info=[self.h0])
			self.s1h, _ = theano.scan(step1, sequences=self.input1, outputs_info=[self.h0])
			self.s2h, _ = theano.scan(step2, sequences=self.input2, outputs_info=[self.h0])

			self.L1 = abs(self.W.sum()) + abs(self.W_in.sum())
			self.L2_sqr = (self.W ** 2).sum() + (self.W_in ** 2).sum()

		self.L1 +=  abs(self.W_log.sum())
		self.L2_sqr += (self.W_log ** 2).sum()

		self.final_layer =  T.concatenate([ self.s0h[ylen0 - 1,:], self.s1h[ylen1 - 1,:], self.s2h[ylen2 - 1,:] ], axis=0)
		self.p_pred = T.nnet.softmax( T.dot( self.final_layer, self.W_log ) + self.blog)
		self.y_sentence = T.argmax( self.p_pred , axis = 1)

		self.updates = {}
		for param in self.params:
			init = np.zeros(param.get_value(borrow=True).shape,
							dtype=theano.config.floatX)
			self.updates[param] = theano.shared(init)

		self.loss = lambda y, ylen0, ylen1, ylen2 : self.nll_multiclass_l(y, ylen0, ylen1, ylen2)

	def nll_multiclass_l(self, y, ylen0, ylen1, ylen2):
		pred_loss = -T.log(self.p_pred[0,y]).sum()
		return  pred_loss

	def debug_final_layer(self, y, ylen0, ylen1, ylen2):
		return (self.p_pred,y,self.final_layer.shape)

	def errors_l(self, y, ylen0, ylen1, ylen2):
		return T.neq(y,self.y_sentence)

	def debug_len(self,ylen0):
		return ylen0

	def debug_lnorm(self):
		return self.L1 + self.L2_sqr

	def predict_instance(self,y, ylen0, ylen1, ylen2):
		return self.y_sentence

class MetaRNN(object):
	def __init__(self, n_in = 7, n_hidden = 50, n_out = 7, learning_rate = 0.01,
				 n_epochs = 100, L1_reg = 0.00001, L2_reg = 0.00001, learning_rate_decay = 1,
				 activation = 'tanh', output_type='softmax',
				 final_momentum = 0.9, initial_momentum = 0.5,
				 momentum_switchover = 5,
				 use_symbolic_softmax = False,
				 lstm_flag = False):
		self.n_in = int(n_in)
		self.n_hidden = int(n_hidden)
		self.n_out = int(n_out)
		self.learning_rate = float(learning_rate)
		self.learning_rate_decay = float(learning_rate_decay)
		self.n_epochs = int(n_epochs)
		self.L1_reg = float(L1_reg)
		self.L2_reg = float(L2_reg)
		self.activation = activation
		self.output_type = output_type
		self.initial_momentum = float(initial_momentum)
		self.final_momentum = float(final_momentum)
		self.momentum_switchover = int(momentum_switchover)
		self.use_symbolic_softmax = use_symbolic_softmax
		self.lstm_flag = lstm_flag
		self.ready()

	def ready(self):
		# input (where first dimension is time)
		self.x0 = T.matrix()
		self.x1 = T.matrix()
		self.x2 = T.matrix()

		self.y = T.scalar(name='y', dtype='int32')

		self.m = T.vector(name='mask-y', dtype='int32')
		# initial hidden state of the RNN
		self.h0 = T.vector()
		# learning rate
		self.lr = T.scalar()

		self.l0 = T.scalar(name='length-y0', dtype='int32')
		self.l1 = T.scalar(name='length-y1', dtype='int32')
		self.l2 = T.scalar(name='length-y2', dtype='int32')

		if self.activation == 'tanh':
			activation = T.tanh
		elif self.activation == 'sigmoid':
			activation = lambda x: 1./(1. + T.exp(-3.75 * x))
		elif self.activation == 'relu':
			activation = lambda x: x * (x > 0)
		elif self.activation == 'cappedrelu':
			activation = lambda x: T.minimum(x * (x > 0), 6)
		else:
			raise NotImplementedError

		self.rnn = LSTM(input0 = self.x0,input1 = self.x1,input2 = self.x2, n_in=self.n_in,
						n_hidden=self.n_hidden, n_out = self.n_out, ylen0 = self.l0, ylen1 = self.l1, ylen2 = self.l2,activation = activation, output_type = self.output_type, lstm_flag = self.lstm_flag)

	def onehot(self,x,numclasses=None):
		""" Convert integer encoding for class-labels (starting with 0 !)
		to one-hot encoding.

        If numclasses (the number of classes) is not provided, it is assumed
        to be equal to the largest class index occuring in the labels-array + 1.
        The output is an array who's shape is the shape of the input array plus
        an extra dimension, containing the 'one-hot'-encoded labels.
		"""
		if x.shape==():
			x = x[None]
		if numclasses is None:
			numclasses = x.max() + 1
		result = np.zeros(list(x.shape) + [numclasses], dtype='int64')
		z = np.zeros(x.shape)
		for c in range(int(numclasses)):
			z *= 0
			z[np.where(x==c)] = 1
			result[...,c] += z
		return result

	def shared_dataset(self, data_xy):
		""" Load the dataset into shared variables """

		data_x, data_y = data_xy
		shared_x = theano.shared(np.asarray(data_x,
											dtype=theano.config.floatX),borrow = True)

		shared_y = theano.shared(np.asarray(data_y,dtype=theano.config.floatX),borrow = True)

		if self.output_type in ('binary', 'softmax'):
			return shared_x, T.cast(shared_y, 'int32')
		else:
			return shared_x, shared_y

	def compute_gradients(self,loss):
		"""Compute gradients of the parameters, given a loss variable"""
		grads = T.grad(loss, self.rnn.params)
		return zip(self.rnn.params, grads)

	def compute_update_rules(self,gradients, lr, momentum):
		"""Compute param updates with momentum"""
		updates = []
		for i in range(len(gradients)):
			w = gradients[i][0]	 # Weight
			g = gradients[i][1]	 # Weight gradient
			g0 = self.rnn.lstm_prev_updates[i] # Previous weight update

			updates.append((g0, momentum * g0 - lr * g))
			updates.append((w, w + g0))
		return updates

	def get_updates(self,cost,l_r,mom):
		if self.lstm_flag :
			updates = self.compute_update_rules(self.compute_gradients(cost), l_r, mom)
		else:
			gparams = []
			for param in self.rnn.params:
				gparam = T.grad(cost, param)
				gparams.append(gparam)

			updates = {}
			for param, gparam in zip(self.rnn.params, gparams):
				weight_update = self.rnn.updates[param]
				upd = mom * weight_update - l_r * gparam
				updates[weight_update] = upd
				updates[param] = param + upd
		return updates

	def debug_grad_norm(self,loss):
		grads = T.grad(loss, self.rnn.params)
		g_total = 0
		for g in grads:
			g_total += abs(g.sum())
		return g_total

	def fit(self, dataset, test_dataset , tag_set, out_file, validation_frequency=100):
		""" Fit model
		"""

		TRAIN,VALIDATION,TEST = dataset
		(X_train,Y_train,M_train_x,M_train_y,L_train) = TRAIN
		(X_val,Y_val,M_val_x,M_val_y,L_val) = VALIDATION
		(X_test,Y_test,M_test_x,M_test_y,L_test) = TEST

		self.interactive = True

		train_s0_x, train_s0_y = self.shared_dataset((X_train, Y_train))
		train_s0_l = theano.shared(np.asarray(L_train, dtype= 'int32'),borrow = True)

		train_s1_x, train_s1_y = self.shared_dataset((X_val, Y_val))
		train_s1_l = theano.shared(np.asarray(L_val, dtype= 'int32'),borrow = True)

		train_s2_x, train_s2_y = self.shared_dataset((X_test, Y_test))
		train_s2_l = theano.shared(np.asarray(L_test, dtype= 'int32'),borrow = True)

		n_train = train_s0_x.get_value(borrow=True).shape[0]

		TRAIN,VALIDATION,TEST = test_dataset
		(X_train,Y_train,M_train_x,M_train_y,L_train) = TRAIN
		(X_val,Y_val,M_val_x,M_val_y,L_val) = VALIDATION
		(X_test,Y_test,M_test_x,M_test_y,L_test) = TEST

		val_s0_x, val_s0_y = self.shared_dataset((X_train, Y_train))
		val_s0_l = theano.shared(np.asarray(L_train, dtype= 'int32'),borrow = True)

		val_s1_x, val_s1_y = self.shared_dataset((X_val, Y_val))
		val_s1_l = theano.shared(np.asarray(L_val, dtype= 'int32'),borrow = True)

		val_s2_x, val_s2_y = self.shared_dataset((X_test, Y_test))
		val_s2_l = theano.shared(np.asarray(L_test, dtype= 'int32'),borrow = True)

		n_val = val_s0_x.get_value(borrow=True).shape[0]

		######################
		# BUILD ACTUAL MODEL #
		######################
		logger.info('... building the model')

		index = T.lscalar('index')	  # index to a case

		cost = self.rnn.loss(self.y, self.l0, self.l1, self.l2) + self.L1_reg * self.rnn.L1 + self.L2_reg * self.rnn.L2_sqr
		gradient = self.debug_grad_norm(cost)

		compute_train_error = theano.function(inputs=[index, ],
											  outputs=self.rnn.errors_l(self.y, self.l0, self.l1, self.l2),
											  givens={
												  self.x0: train_s0_x[index],
												  self.x1: train_s1_x[index],
												  self.x2: train_s2_x[index],
												  self.y : train_s0_y[index][0],
												  self.l0 : train_s0_l[index],
												  self.l1 : train_s1_l[index],
												  self.l2 : train_s2_l[index]},
											  mode=mode)

		compute_val_error = theano.function(inputs=[index, ],
											  outputs=self.rnn.errors_l(self.y, self.l0, self.l1, self.l2),
											  givens={
												  self.x0: val_s0_x[index],
												  self.x1: val_s1_x[index],
												  self.x2: val_s2_x[index],
												  self.y : val_s0_y[index][0],
												  self.l0 : val_s0_l[index],
												  self.l1 : val_s1_l[index],
												  self.l2 : val_s2_l[index]},
											  mode=mode)

		l_norm = theano.function(inputs=[],outputs=self.rnn.debug_lnorm(),givens={},mode=mode)

		get_final_layer = theano.function(inputs=[index,],
									   outputs=self.rnn.debug_final_layer(self.y, self.l0, self.l1, self.l2),
									   givens={
										   self.x0: train_s0_x[index],
										   self.x1: train_s1_x[index],
										   self.x2: train_s2_x[index],
										   self.y : train_s0_y[index][0],
										   self.l0 : train_s0_l[index],
										   self.l1 : train_s1_l[index],
										   self.l2 : train_s2_l[index]},
									   mode=mode)
		###! here
		predict_test = theano.function(inputs=[index,],
									   outputs=self.rnn.predict_instance(self.y, self.l0, self.l1, self.l2),
									   givens={
										   self.x0: val_s0_x[index],
										   self.x1: val_s1_x[index],
										   self.x2: val_s2_x[index],
										   self.y : val_s0_y[index][0],
										   self.l0 : val_s0_l[index],
										   self.l1 : val_s1_l[index],
										   self.l2 : val_s2_l[index]},
									   mode=mode)

		self.updater = update_list.Adadelta(lr = self.learning_rate)
		updates = self.updater.get_updates(self.rnn.params, cost)

		###! here
		train_model = theano.function(inputs=[index,],
									  outputs=[cost,gradient],
									  updates=updates,
									  givens={
										  self.x0: train_s0_x[index],
										  self.x1: train_s1_x[index],
										  self.x2: train_s2_x[index],
										  self.y : train_s0_y[index][0],
										  self.l0 : train_s0_l[index],
										  self.l1 : train_s1_l[index],
										  self.l2 : train_s2_l[index]},
									  mode=mode)

		###############
		# TRAIN MODEL #
		###############
		logger.info('... training')
		epoch = 0

		while (epoch < self.n_epochs):
			epoch = epoch + 1

			epoch_gradient = np.array([])
			batch_loss = []
			id_dict = defaultdict(int)
			for idx in xrange(n_train):
				example_cost,example_gradient = train_model(idx)
				example_final_layer = get_final_layer(idx)
				_,y,_= example_final_layer

				y = int(y.astype(int))
				id_dict [y] += 1

				batch_loss.append(example_cost)
				epoch_gradient = np.append(epoch_gradient,[example_gradient])
				example_final_layer = get_final_layer(0)
				iter = (epoch - 1) * n_train + idx + 1

				if iter % validation_frequency == 0:

					train_losses = [compute_train_error(i) for i in xrange(n_train)]
					this_train_loss = np.mean(train_losses)

#					val_losses = [compute_val_error(i) for i in xrange(n_val)]
					this_val_loss = 0 #np.mean(val_losses)

					logger.info('epoch %i/%i, seq %i/%i, tr error %.3f val error %.3f loss : %.3f lnorm %.3f gradnorm %.3f ' % \
						(epoch, self.n_epochs, idx + 1, n_train, this_train_loss, this_val_loss, np.mean(batch_loss), l_norm(), epoch_gradient.mean()))

		print >> sys.stderr, "predicting test file"
		out = open(out_file,'w')
		for i in xrange(n_val):
			pred = predict_test(i)
			print >> out, tag_set[pred[0]]

def runData(data_file):
	try:
		(dim,num_tag,tag_set,dataset) = pickle.load(gzip.open(p.benchmark))
	except:
		print >> sys.stderr, "data file",p.benchmark ,"cannot be read"
		quit(1)

	try:
		(_,_,_,test_dataset) = pickle.load(gzip.open(p.test_benchmark))
	except:
		print >> sys.stderr, "test file",p.test_benchmark,"cannot be read"
		quit(1)

	model = MetaRNN(n_in= dim, n_hidden= p.n_hidden, n_out= num_tag,
					learning_rate= p.learning_rate, learning_rate_decay= p.learning_rate_decay,
					n_epochs= p.n_epochs, activation= p.activation,
					L1_reg = p.L1_reg, L2_reg = p.L2_reg,
					lstm_flag = p.lstm_flag)
	out_file = p.out_file
	if out_file == "":
		lstm = 'out'
		if p.lstm_flag:
			lstm = ''
		out_file = 'test.'+'_'.join(['b-'+p.benchmark.split('.')[0],'e-'+str(p.n_epochs),'l-'+str(p.learning_rate),'h-'+str(p.n_hidden),'a-'+p.activation,'with'+lstm+'lstm'])

	model.fit(dataset, test_dataset, tag_set, out_file, validation_frequency= p.validation_frequency)

if __name__ == "__main__":
	logging.basicConfig(level=logging.INFO)
	np.random.seed(seed = 0)
	parser = get_parser()
	p = parser.parse_args()
	runData(p)


from mymodule import *
import numpy as np
import matplotlib.pyplot as plt

# script parameters
losscoeffs = [50, 4, -6, 0, 1]
display_loss_function = True
display_graphs = True
run_gradient_descent = False
run_momentum = False
run_Adam = True
# -------------------------------------------

# define loss function
lossfunc = LossFunc(losscoeffs)

# display the loss function
xstart, xstop, npoints = -3, 3, 100
x = np.linspace(xstart, xstop, npoints)
if display_loss_function:
    lossfunc.display(x)

# define optimization algorithm
initial_weight = 3
min_step = 0.001  # early stop
max_iterations = 100

# create gradient descent object
learn_rate = 0.05
grad = GradientDescent(lossfunc=lossfunc, learnrate=learn_rate, weight=initial_weight,
                       minstep=min_step, maxiterations=max_iterations)

# run the optimization algorithm
if run_gradient_descent:
    grad.run()
    grad.printlog()
    if display_graphs:
        grad.plotlog()

# create momentum object
learn_rate = 0.05
momentum_coeff = 0.9
moment = Momentum(lossfunc=lossfunc, learnrate=learn_rate, weight=initial_weight,
                  minstep=min_step, maxiterations=max_iterations, momentcoeff=momentum_coeff)

# run the optimization algorithm
if run_momentum:
    moment.run()
    moment.printlog()
    if display_graphs:
        moment.plotlog()

# create Adam object
learn_rate = 1
beta = 0.9
adam = Adam(lossfunc=lossfunc, learnrate=learn_rate, weight=initial_weight, minstep=min_step,
            maxiterations=max_iterations, momentcoeff=momentum_coeff, beta=beta)

if run_Adam:
    adam.run()
    adam.printlog()
    if display_graphs:
        adam.plotlog()

# show all plots if exist
plt.show(block=True)


from ...data import Data

from bson import json_util, ObjectId
import json

class Model:
    def __init__(self):
        self.id = None
        self.client = Data.db.client
        self.db = self.client["binance_gtm"]
        self.spot_wallet = self.db["spot_wallet"]
        self.trade_history = self.db["trade_history"]

    # = = = = = = = = = = = = = = = = = = = = = = = = = =
    #            INSERT MODEL TO DATABASE
    # = = = = = = = = = = = = = = = = = = = = = = = = = =

    def insert(self):
        pass

    # = = = = = = = = = = = = = = = = = = = = = = = = = =
    #                     GET MODEL
    # = = = = = = = = = = = = = = = = = = = = = = = = = =

    @staticmethod
    def get(query, col):
        """
        This function gets single record from database and return query result.
        @params
            - query
            - column_name

        @return
            - query_result : Query
        """
        db = Data.db.client["binance_gtm"]

        query_result = db[col].find_one(query)

        return query_result

    # = = = = = = = = = = = = = = = = = = = = = = = = = =
    #                  GENERATE A JSON
    # = = = = = = = = = = = = = = = = = = = = = = = = = =

    def to_json(self):
        return json.dumps(self.__dict__)

    # = = = = = = = = = = = = = = = = = = = = = = = = = =
    #                    SAVE MODEL
    # = = = = = = = = = = = = = = = = = = = = = = = = = =

    def save(self):

        """
        This function changes pre-object which is on database with new object.
        @params
            - None
        @return
            - update_result : pymongo.results.UpdateResult
        """

        column_name = self.__class__.column_name

        col = self.__getattribute__(column_name)

        query = {"_id": ObjectId(self.id)}

        updated = {"$set": self.to_json()}

        try:
            return col.update_one(query, updated)
        except:
            return None

    # = = = = = = = = = = = = = = = = = = = = = = = = = =
    #                  GET ALL MODELS
    # = = = = = = = = = = = = = = = = = = = = = = = = = =

    @classmethod
    def get_all(cls, query={}):

        """
        This function changes pre-object which is on database with new object.
        @params
            - None
        @return
            - update_result : pymongo.results.UpdateResult
        """

        column_name = cls.column_name

        col = Model().__getattribute__(column_name)

        all_docs = col.find(query)

        return all_docs

    # = = = = = = = = = = = = = = = = = = = = = = = = = =
    #            GENERATE A INSTANCE FROM JSON
    # = = = = = = = = = = = = = = = = = = = = = = = = = =

    @classmethod
    def from_json(cls, cursor):

        """
        This function converts string json to dict. After that,
        create single instance by using values which is inside this dict
        @params
            - cls : class (auto provided)
            - cursor : CursorObject
        @return
            - object : (generated given classname)
        """

        json_str = json.dumps(
            cursor, sort_keys=True, indent=4, default=json_util.default
        )

        json_data = json.loads(json_str)

        json_data["_id"] = json_data["_id"]["$oid"]

        return cls(**json_data)

    # = = = = = = = = = = = = = = = = = = = = = = = = = =
    #       GENERATE MULTIPLE INSTANCES FROM JSONS
    # = = = = = = = = = = = = = = = = = = = = = = = = = =

    @classmethod
    def from_jsons(cls, cursor):

        """
        This function converts string json to dict. After that,
        create multiple instances by using values which is inside this dict
        @params
            - cls : class (auto provided)
            - cursor : CursorObject
        @return
            - array : (an array which is inside instances)
        """

        if cursor != None:
            clist = []

            for cs in cursor:
                clist.append(cls(**cs))

            return clist

        else:
            return None


import os
import json
import collections
import xml.dom.minidom
import urllib.request
import urllib.parse
import argparse

parser = argparse.ArgumentParser()
parser.add_argument('--oaixml')
parser.add_argument('--iconclass', default = 'iconclass')
parser.add_argument('--cut', type = int, default = 100)
args = parser.parse_args()
if not os.path.exists(args.iconclass):
	os.makedirs(args.iconclass)

stats = collections.defaultdict(int)

for i, filename in enumerate(os.listdir(args.oaixml)):
	doc = xml.dom.minidom.parse(os.path.join(args.oaixml, filename))
	for rec in doc.getElementsByTagName('dc:subject'):
		iconclass_ = rec.firstChild.data
		iconclass = iconclass_[:args.cut]
		if iconclass.count('(') != iconclass.count(')'):
			iconclass_ += ')'
			iconclass = iconclass_[:iconclass_.index(')') + 1]
		stats[iconclass.strip()] += 1

def iconclass_describe(cat, cache):
	cachepath = os.path.join(cache, cat + '.json')
	if not os.path.exists(cachepath):
		with open(cachepath, 'w') as f:
			url = 'http://www.iconclass.org/{}.json'.format(urllib.parse.quote(cat))
			json.dump(json.load(urllib.request.urlopen(url)), f)
	return json.load(open(cachepath))['txt']['en']

top = list(reversed(sorted([(v, k) for k, v in stats.items()])))[:200]
print('\n'.join('{: 6}\t{}\t{: <40}\t{: <40}\t{}'.format(k, v, iconclass_describe(v[:1], args.iconclass), iconclass_describe(v[:2], args.iconclass), iconclass_describe(v, args.iconclass)) for k, v in top))


# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.db import models, migrations

class Migration(migrations.Migration):

    dependencies = [
    ]

    operations = [
        migrations.CreateModel(
            name='Task',
            fields=[
                ('id', models.AutoField(verbose_name='ID', auto_created=True, primary_key=True, serialize=False)),
                ('task_title', models.CharField(verbose_name='Название задания', max_length='256')),
                ('task_status', models.BooleanField(verbose_name='Статус задания', default=False)),
            ],
            options={
                'db_table': 'tasks',
            },
        ),
    ]


import asyncio

from confluent_kafka import Consumer

async def run_kafka_consumer(topic_name):
    consumer = Consumer({
        'bootstrap.servers': 'PLAINTEXT://localhost:9092',
        'group.id': '0',
    })

    consumer.subscribe([topic_name])

    while True:
        messages = consumer.consume()

        for message in messages:
            if message is None:
                print('Message not found')
            elif message.error() is not None:
                print(f'Error: {message.error()}')
            else:
                print(f'{message.value()}\n')

        await asyncio.sleep(1.0)

def consume():
    try:
        asyncio.run(run_kafka_consumer('com.crime.police_call'))

    except KeyboardInterrupt as e:
        print("Shutting down...")

if __name__ == '__main__':
    consume()


from pathlib import Path
from torch.utils.data import DataLoader
import torch
import numpy as np
from torch.nn import CrossEntropyLoss
from pgdataset.s3_handcraft import PgdHandcraft
from constants.enum_keys import PG
from models.gesture_recognition_model import GestureRecognitionModel
from torch import optim
from constants import settings

class Trainer:
    def __init__(self, is_unittest=False):
        self.is_unittest = is_unittest
        self.batch_size = 1
        self.clip_len = 5*7
        pgd = PgdHandcraft(Path.home() / 'intentlong', True, (512, 512), clip_len=self.clip_len)
        self.data_loader = DataLoader(pgd, batch_size=self.batch_size, shuffle=False, num_workers=settings.num_workers)
        self.model = GestureRecognitionModel(batch=self.batch_size)
        self.model.train()
        self.loss = CrossEntropyLoss()
        self.opt = optim.Adam(self.model.parameters(), lr=1e-3)

    def train(self):
        step = 0
        self.model.load_ckpt()
        for epoch in range(100000):
            for ges_data in self.data_loader:
                features = torch.cat((ges_data[PG.BONE_LENGTH], ges_data[PG.BONE_ANGLE_COS],
                                      ges_data[PG.BONE_ANGLE_SIN]), dim=2)
                features = features.permute(1, 0, 2)
                features = features.to(self.model.device, dtype=torch.float32)
                h0, c0 = self.model.h0(), self.model.c0()

                _, h, c, class_out = self.model(features, h0, c0)
                target = ges_data[PG.GESTURE_LABEL]
                target = target.to(self.model.device, dtype=torch.long)
                target = target.permute(1, 0)
                target = target.reshape((-1))
                loss_tensor = self.loss(class_out, target)
                self.opt.zero_grad()
                loss_tensor.backward()
                self.opt.step()

                if step % 100 == 0:
                    print("Step: %d, Loss: %f" % (step, loss_tensor.item()))
                if step % 5000 == 0 and step != 0:
                    self.model.save_ckpt()
                if self.is_unittest:
                    break
                step = step + 1
            if self.is_unittest:
                break
#p1 = Trainer()
#p1.train()


import re
import time
from dateutil import parser
from datetime import datetime
from .parser import parse_page
from .utils import get_soup
from .utils import report_dateformat
from .utils import user_dateformat
from .utils import commentary_dateformat
from .utils import strf_to_datetime

def is_matched(url):
    for pattern in patterns:
        if pattern.match(url):
            return True
    return False

patterns = [
    re.compile('https://www.heritage.org/.+/commentary/.+'),
    re.compile('https://www.heritage.org/.+/report/.+')
    ]

url_yield = 'https://www.heritage.org/search?contains={}&type%5B%5D=commentary&type%5B%5D=report&range_start={}&range_end={}&date_offset=&page={}'

def yield_latest_article(query, begin_date, end_date, max_num=10, sleep=1.0):
    """
    Artuments
    ---------
    begin_date : str
        eg. 2018-07-01
    end_date :str
        eg. 2019-03-31
    max_num : int
        Maximum number of news to be scraped
    sleep : float
        Sleep time. Default 1.0 sec

    It yields
    ---------
    news : json object
    """

    # prepare parameters
    end_page = 72
    n_news = 0
    outdate = False
    links_all= []
    for page in range(0, end_page+1):

        # get urls

        url = url_yield.format(query, begin_date, end_date, page)
        soup = get_soup(url)
        sub_links = soup.find_all('a', class_= 'result-card__title js-hover-target')
        links = ['https://www.heritage.org' + i.attrs['href'] for i in sub_links]
        links_all += links
        links_all = [url for url in links_all if is_matched(url)]

        # scrap
        for url in links_all:
            news_json = parse_page(url)
            # yield
            yield news_json

            # check number of scraped news
            n_news += 1
            if n_news >= max_num:
                break
            time.sleep(sleep)

def get_article_urls(query, begin_date, end_date, max_num=10, sleep=1.0):
    """
    Arguments
    ---------
    begin_page : int
        Default is 1
    end_page : int
        Default is 3
    verbose : Boolean
        If True, print current status

    Returns
    -------
    links_all : list of str
        List of urls
    """
    end_page = 72
    n_news = 0
    outdate = False

    links_all = []
    for page in range(0, end_page+1):
        url = url_yield.format(query, begin_date, end_date, page)
        soup = get_soup(url)
        sub_links = soup.find_all('a', class_= 'result-card__title js-hover-target')
        links = ['https://www.heritage.org' + i.attrs['href'] for i in sub_links]
        links_all += links
        links_all = [url for url in links_all if is_matched(url)]

    return links_all


"""
This file defines actions, i.e. functions the URLs are mapped into
The @action(path) decorator exposed the function at URL:

    http://127.0.0.1:8000/{app_name}/{path}

If app_name == '_default' then simply

    http://127.0.0.1:8000/{path}

If path == 'index' it can be omitted:

    http://127.0.0.1:8000/

The path follows the bottlepy syntax.

@action.uses('generic.html')  indicates that the action uses the generic.html template
@action.uses(session)         indicates that the action uses the session
@action.uses(db)              indicates that the action uses the db
@action.uses(T)               indicates that the action uses the i18n & pluralization
@action.uses(auth.user)       indicates that the action requires a logged in user
@action.uses(auth)            indicates that the action requires the auth object

session, db, T, auth, and tempates are examples of Fixtures.
Warning: Fixtures MUST be declared with @action.uses({fixtures}) else your app will result in undefined behavior
"""

import uuid

from py4web import action, request, abort, redirect, URL, Field
from py4web.utils.form import Form, FormStyleBulma
from py4web.utils.url_signer import URLSigner
from pydal.validators import *

from yatl.helpers import A
from . common import db, session, T, cache, auth, signed_url

url_signer = URLSigner(session)

# The auth.user below forces login.
@action('index', method='GET')
@action.uses('index.html', auth.user,session,db)
def index():
	#passes user variable to allow for signout button to display while signed in.
	user=auth.current_user
	#query that selects the contacts under the email of the current logged in user
	rows = db(db.person.user_email==user.get('email')).select().as_list()
	for row in rows:

		s=db(db.phone.owner_id==row.get('id')).select(db.phone.phone_number,db.phone.phone_name)

		row["phone_number"]=s

		pass
	return dict(rows=rows, url_signer=url_signer, user=user)

@action('add_contact', method=['GET', 'POST'])
@action.uses('contact_form.html', auth.user, session, db)
def add_contact():
    form = Form(db.person, csrf_session=session, formstyle=FormStyleBulma)
    if form.accepted:
        # We always want POST requests to be redirected as GETs.
        redirect(URL('index'))
    return dict(form=form)

@action('edit_contact/<contact_id>', method=['GET', 'POST'])
@action.uses('contact_form.html', auth.user, session, db)
def edit_contact(contact_id=None):

    p = db.person[contact_id]
    pCreator=p.user_email
    #checks if actually a entry
    if p is None:
        redirect(URL('index'))
    #checks if current user is editing their own contact
    if pCreator!=auth.current_user.get('email'):
    	redirect(URL('index'))

    form = Form(db.person, record=p, deletable=False, csrf_session=session, formstyle=FormStyleBulma)
    if form.accepted:
        # We always want POST requests to be redirected as GETs.
        redirect(URL('index'))
    return dict(form=form)

@action('delete_contact/<contact_id>', method=['GET','POST'])
@action.uses('index.html',auth.user, session, db, url_signer.verify())
def delete_contact(contact_id=None):
    p = db.person[contact_id]
    pCreator=p.user_email
    #check if actually a entry
    if p is None:
        redirect(URL('index'))
    #if user is deleting a contact that isnt theirs
    if pCreator!=auth.current_user.get('email'):
    	redirect(URL('index'))
    else:
        db(db.person.id == contact_id).delete()
        redirect(URL('index'))

@action('edit_phones/<contact_id>', method=['GET', 'POST'])
@action.uses('edit_phones.html', auth.user, session, db)
def edit_phones(contact_id=None):

    #passes user variable to allow for signout button to display while signed in.
	user=auth.current_user
	#pulls data from person table
	contactInfo=db(db.person.id==contact_id).select()

	if auth.current_user.get('email')!=contactInfo[0].user_email:
		redirect(URL('index'))
	#query that selects the contacts under the email of the current logged in user
	rows = db((db.person.user_email==user.get('email')) & (db.person.id==contact_id) & (db.phone.owner_id==contact_id)).select().as_list()

	return dict(rows=rows,contactInfo=contactInfo, url_signer=url_signer, user=user)

@action('add_phone/<contact_id>', method=['GET', 'POST'])
@action.uses('phone_form.html', auth.user, session, db)
def add_phone(contact_id=None):
	contactInfo=db(db.person.id==contact_id).select()

	if auth.current_user.get('email')!=contactInfo[0].user_email:
		redirect(URL('index'))

	form = Form([Field('phone_number', requires=IS_NOT_EMPTY()),Field('phone_name',requires=IS_NOT_EMPTY())],csrf_session=session,formstyle=FormStyleBulma)
	if form.accepted:
		# We always want POST requests to be redirected as GETs.

		db.phone.insert(phone_number=form.vars["phone_number"], phone_name=form.vars["phone_name"], owner_id=contact_id)
		redirect(URL('edit_phones',contact_id))
	return dict(form=form, contactInfo=contactInfo)

@action('edit_phone/<contact_id>/<phone_id>', method=['GET', 'POST'])
@action.uses('phone_form.html', auth.user, session, db)
def edit_phone(contact_id=None, phone_id=None):
	contactInfo=db(db.person.id==contact_id).select()

	if auth.current_user.get('email')!=contactInfo[0].user_email:
		redirect(URL('index'))
	p = db.phone[phone_id]

	form = Form([Field('phone_number',requires=IS_NOT_EMPTY()),Field('phone_name',requires=IS_NOT_EMPTY())],record=dict(phone_number=p.phone_number, phone_name=p.phone_name),deletable=False, csrf_session=session,formstyle=FormStyleBulma)
	if form.accepted:

		#modeled after documentation on update
		myset = db(db.phone.id==phone_id)
		myset.update(phone_number=form.vars["phone_number"], phone_name=form.vars["phone_name"], owner_id=contact_id)
		redirect(URL('edit_phones',contact_id))
	return dict(form=form,contactInfo=contactInfo)

@action('delete_phone/<contact_id>/<phone_id>', method=['GET','POST'])
@action.uses('edit_phones.html',auth.user, session, db, url_signer.verify())
def delete_phone(contact_id=None,phone_id=None):
    p = db.phone[phone_id]
    contactInfo=db(db.person.id==contact_id).select()

    #check if actually a entry
    if p is None:
        redirect(URL('edit_phones',contact_id))
    #if user is deleting a contact that isnt theirs
    if contactInfo[0].user_email!=auth.current_user.get('email'):
    	redirect(URL('edit_phones',contact_id))
    else:
        db(db.phone.id == phone_id).delete()
        redirect(URL('edit_phones',contact_id))


# -*- coding: utf-8 -*-
"""
Routine to detect last date a transmitter reported
originally coded by JiM in July 2016
Note: It reads output of "getap2s.py" which is /net/pubweb_html/drifter/emolt.dat
Note: In May 2017, I added emolt_ap3.dat
Note: In May 2018, I started adding "day last fished"
Note: In Jan 2019, I added DOPPIO (currently subtracts 4 days to pick up file needed)
Note: In Feb 2019, I made a temp file "lastfix_temp.html" until routine was complete
Note: In Mar 2019, I added GOMOFS and fixed "getlastwifi" and revised gomofs_modules method
Note: In Mar 2019, I added "mindistfromharbor" to avoid request from dock to appear in lastfix website
Note: In Apr 2019, I added a) a check on depth compared to NGDC depth and b) a fix to check harborlist
Note: In May 2019, I fixed the getlastwifi function
Note: In Aug 2020, I started implementing the "datatables" sort function
Note: In Jan 2021, I moved to my new laptop at home, made ready for Python 3,
"""

from datetime import datetime as dt
from datetime import timedelta
from dateutil import parser
import pandas as pd
import pytz
import glob
import numpy as np
import netCDF4
import os
from conversions import c2f,dd2dm # converts celcius to fahrenheight and decimal degrees to degrees-minutes format
#import zlconversions as zl # Lei Zhao's conversion routines
#import gomofs_modules_zl2 as gomofs_modules # lei Zhao's GOMOFS functions
import gomofs_modules # lei Zhao's final quick version GOMOFS functions
import doppio_modules
import warnings
from pandas.core.common import SettingWithCopyWarning
warnings.simplefilter(action="ignore", category=SettingWithCopyWarning)
warnings.simplefilter(action="ignore", category=RuntimeWarning)
#
# HARDCODES:
maxdiff=10. # maximum model error to report
mindistfromharbor=0.4 # mimimum distance in miles from harbor to be considered real
mindist_allowed=0.4 # minimum distance from nearest NGDC depth in km

# function to get vessel_name and probe serial number given vessel_number
def getclim(lat1=43.,lon1=-73.0,yrday=dt.now().strftime('%j').lstrip('0'),var='Bottom_Temperature/BT_'):
  # gets climatology of Bottom_Temperature  (note:Surface_Temperature, Bottom_Salinity, or Surface_Salinity also available)
  # as calculated by Chris Melrose from 30+ years of NEFSC CTD data on the NE Shelf
  # where "lat1", "lon1", and "yrday" are the position and yearday of interest
  # where "var" is the variable of interest (usually Bottom_Temperature)
  inputdir='/net/data5/jmanning/clim/' # hardcoded climatology directory name "/home/pi/clim/"
  dflat=pd.read_csv(inputdir+'LatGrid.csv',header=None)
  dflon=pd.read_csv(inputdir+'LonGrid.csv',header=None)
  lat=np.array(dflat[0])   # gets the first col (35 to 45)
  lon=np.array(dflon.ix[0])# gets the first row (-75 to -65)
  clim=pd.read_csv(inputdir+var+str(int(yrday))+'.csv',header=None) # gets bottom temp for this day of year
  idlat = np.abs(lat - lat1).argmin() # finds the neareast lat to input lat1
  idlon = np.abs(lon - lon1).argmin() # finds the neareast lon to input lon1
  return clim[idlon][idlat]

def getvesselname(vessel_number):
  #df=pd.read_csv('/net/data5/jmanning/drift/codes_temp.dat',sep='\s+',header=None)
  df=pd.read_csv('codes_temp.dat',sep='\s+',header=None)
  vn=df[df[0]==vessel_number][3].values[0] #
  sn=df[df[0]==vessel_number][4].values[-1]# gets the last one
  form=df[df[0]==vessel_number][5].values[-1]# fixed or mobile as "f" or "m"
  #sn=sn[2:4]# just use the last two characters
  return vn,sn,form

def getlastwifi(sn):
  #if (sn[0:2]=='7a') or (sn[0:2]=='4d'): # case of Lowell Instruments
  if (sn[0]!='1'): # case of Lowell Instruments where serial numbers starting with "1" are aquatec (change made 3/13/19)
    #filenames=glob.glob('/net/data5/jmanning/li/Matdata/temporary/li_*.csv')
    #filenames=glob.glob('/home/jmanning/py/aq_main/aqmain_and_raw_check/raw_data_check/result/Matdata/li_*.csv') # change made after lei Zhao's method was implemented in Aug 2019
    filenames=glob.glob('li_*.csv') # change made after lei Zhao's method was implemented in Aug 2019
    #mrf=[s for s in filenames if "li_"+sn[2:4] in s]
    #mrf=[s for s in filenames if "li_"+sn in s] # change made 3/13/2019
    #mrf=[s for s in filenames if "li_"+sn[-5:-1] in s] # change made 4/9/2019 to account for some new sn having 6 characters but only four in filenames
    mrf=[s for s in filenames if "li_"+sn[-4:] in s] # change made 5/16/2019
  else: # case of Aquatec
    filenames=glob.glob('/home/jmanning/py/api/aqu_data/Logger_sn_*.csv')
    mrf=[s[-19:] for s in filenames if "Logger_sn_"+sn in s]
    #print sn
  if len(mrf)==0:# empty string
    last_wifi_date='never'
  else:
    mrf=max(mrf)
    #print mrf
    try:
      mrf=mrf.split('Matdata/')[1] # this gets the last file according to date
    except:
      pass

    if sn[0]!='1': # case of Lowell Instruments
      i=[x for x, v in enumerate(mrf) if v == '_'][1]# finds index of 2nd underscore in the filename

      last_wifi_date=dt(int(mrf[i+1:i+5]),int(mrf[i+5:i+7]),int(mrf[i+7:i+9]),0,0,0).strftime("%b %d,%Y")
    else: # aquatec case
      #print mrf
      last_wifi_date=dt(int(mrf[0:4]),int(mrf[4:6]),int(mrf[6:8]),0,0,0).strftime("%b %d,%Y")
  return last_wifi_date

def getlastfished(vn,mindistfromharbor):
   # function to determine when the last date the vessel "reported" away from the dock
   # assuming the dock position is entered in "harborlist.txt" file
   #df=pd.read_csv('/net/pubweb_html/drifter/emolt_ap3_reports.dat',sep=',',header=None) # this file is created during the "getap3.py" run
   #df=pd.read_csv('emolt_ap3_reports.dat',sep=',',header=None) # this file is created during the "getap3.py" run
   df=pd.read_csv('http://nefsc.noaa.gov/drifter/emolt_ap3_reports.dat',sep=',',header=None) # this file is created during the "getap3.py" run
   df1=df[df[0]==vn] # this assumes the vessel names in "codes_temp.dat" are exactly the same as those on the AssetLink site but with the "F/V " prefix removed
   if len(df1)>0:
     yorn=[] # yes  or no near a harbor
     for k in range(len(df1)): # get positions away from the dock
        [la,lo]=dd2dm(list(df1[2])[k],list(df1[3])[k])# converts to ddmm format
        yorn.append(gps_compare_JiM(la,lo,mindistfromharbor))
     df1[4]=yorn # this is a list of yes and no's
     df2=df1[df1[4]=='no'] # forms a new dataframe with only those positions away from the dock
     list_of_dates=list(df2[1])
     if len(list_of_dates)==0:
        last_fished_date='unknown'
     else:
        last_fished_date=parser.parse(max(list_of_dates)).strftime("%b %d,%Y") # last date away from the dock
   else:
     last_fished_date='unknown'
   if vn=='Lady_Jane':
     last_fished_date='retired'
   return last_fished_date# return the last date the vessel fished

def gps_compare_JiM(lat,lon,mindistfromharbor): #check to see if the boat is in the harbor derived from Huanxin's "wifipc.py" functions
    # function returns yes if this position is with "mindistfromharbor" miles of a dock
    #file='/home/jmanning/py/harborlist.txt' # has header line lat, lon, harbor
    file='harborlist.txt' # has header line lat, lon, harbor
    df=pd.read_csv(file,sep=',')
    indice_lat=[i for i ,v in enumerate(abs(np.array(df['lat'])-lat)<mindistfromharbor) if v]
    indice_lon=[i for i ,v in enumerate(abs(np.array(df['lon'])-lon)<mindistfromharbor) if v]
    harbor_point_list=[i for i, j in zip(indice_lat,indice_lon) if i==j]
    if len(harbor_point_list)>0:
       near_harbor='yes'
    else:
       near_harbor='no'
    return near_harbor #yes or no

def get_depth(loni,lati,mindist_allowed):
    # routine to get depth (meters) using vol1 from NGDC
    if lati>=40.:
        url='https://www.ngdc.noaa.gov/thredds/dodsC/crm/crm_vol1.nc'
    else:
        url='https://www.ngdc.noaa.gov/thredds/dodsC/crm/crm_vol2.nc'
    nc = netCDF4.Dataset(url).variables
    lon=nc['x'][:]
    lat=nc['y'][:]
    xi,yi,min_dist= nearlonlat_zl(lon,lat,loni,lati)
    if min_dist>mindist_allowed:
      depth=np.nan
    else:
      depth=nc['z'][yi,xi]
    return depth#,min_dist

def nearlonlat_zl(lon,lat,lonp,latp): # needed for the next function get_FVCOM_bottom_temp
    """
    used in "get_depth"
    """
    # approximation for small distance
    cp=np.cos(latp*np.pi/180.)
    dx=(lon-lonp)*cp
    dy=lat-latp
    xi=np.argmin(abs(dx))
    yi=np.argmin(abs(dy))
    min_dist=111*np.sqrt(dx[xi]**2+dy[yi]**2)
    return xi,yi,min_dist

def nearlonlat(lon,lat,lonp,latp): # needed for the next function get_FVCOM_bottom_temp
    """
    i=nearlonlat(lon,lat,lonp,latp) change
    find the closest node in the array (lon,lat) to a point (lonp,latp)
    input:
        lon,lat - np.arrays of the grid nodes, spherical coordinates, degrees
        lonp,latp - point on a sphere
        output:
            i - index of the closest node
            For coordinates on a plane use function nearxy
            Vitalii Sheremet, FATE Project
    """
    cp=np.cos(latp*np.pi/180.)
    # approximation for small distance
    dx=(lon-lonp)*cp
    dy=lat-latp
    dist2=dx*dx+dy*dy
    i=np.argmin(dist2)
    return i

def get_FVCOM_bottom_temp(lati,loni,dtime,layer): # gets modeled temp using GOM3 forecast
        '''
        Taken primarily from Rich's blog at: http://rsignell-usgs.github.io/blog/blog/2014/01/08/fvcom/ on July 30, 2018
        where lati and loni are the position of interest, dtime is the datetime, and layer is "-1" for bottom
        '''
        #urlfvcom = 'http://www.smast.umassd.edu:8080/thredds/dodsC/fvcom/hindcasts/30yr_gom3'
        #urlfvcom = 'http://www.smast.umassd.edu:8080/thredds/dodsC/FVCOM/NECOFS/Forecasts/NECOFS_FVCOM_OCEAN_MASSBAY_FORECAST.nc'
        urlfvcom = 'http://www.smast.umassd.edu:8080/thredds/dodsC/FVCOM/NECOFS/Forecasts/NECOFS_GOM3_FORECAST.nc'
        nc = netCDF4.Dataset(urlfvcom).variables
        #first find the index of the grid
        lat = nc['lat'][:]
        lon = nc['lon'][:]
        inode = nearlonlat(lon,lat,loni,lati)
        #second find the index of time
        time_var = nc['time']
        itime = netCDF4.date2index(dtime,time_var,select='nearest')# where startime in datetime
        if itime==0:
            print('time is before fvcom forecast')
            return np.nan
        else:
            return nc['temp'][itime,layer,inode]

#MAIN PROGRAM

# open output html file and write header line
outfile=open('lastfix_temp.html','w')
#outfile=open('/net/pubweb_html/drifter/lastfix_temp.html','w')
#outfile=open('/net/pubweb_html/drifter/lastfix_temp4.html','w')
#outfile=open('/net/newfish_www/html/nefsc/emolt/lastfix_temp.html','w')

outfile.write('\n')
outfile.write('\n')
outfile.write('\n')
outfile.write('<link rel="stylesheet" type="text/css" href="https://cdn.datatables.net/1.10.21/css/jquery.dataTables.min.css">\n')
outfile.write('\n')

outfile.write('
commercial fishing vessels telemetering realtime bottom temperatures from the Northeast Shelf
\n')
outfile.write('
\n')
outfile.write('

vessel	date_last_telemetered	last_obs (degF)	FVCOM	DOPPIO	GOMFS (degF)	Climatology	#Transmitted	probe SN	last_wifi_upload	last_fished

\n')
outfile.write('
\n')

# read input file assuming that emolt_ap3.dat has been appended to this at 10, 40 minutes after even hour
df=pd.read_csv('http://emolt.org/emoltdata/emolt.dat',sep='\s+',header=None)
#df=pd.read_csv('https://nefsc.noaa.gov/drifter/emolt.dat',sep='\s+',header=None)
#df=pd.read_csv('https://apps-nefsc.fisheries.noaa.gov/drifter/emolt.dat',sep='\s+',header=None)           # assetlink results and assumes rockblock hauls already appended to AL by "getap3.py" 8/19/2020

# create a datetime for all fixes
datet=[]
for k in range(len(df)): # for each fix
  datet.append(dt(df[16][k],df[2][k],df[3][k],df[4][k],df[5][k]))
df['datet']=datet

# for each vessel find the last datetime and write to file
ves=list(set(df[0])) # list of unique vessels numbers

sumnum=0
anomaly,moderror,moderror_doppio,moderror_gomofs=[],[],[],[]# intitialize a few variables for appending later
#for j in range(len(ves)): # for each vessel
for j in [0]:
  print(ves[j])
  df1=df[df[0]==ves[j]]# gets all the records for this one vessel in new dataframe "df1"
  # added the following 6 lines 29 Mar 2019
  yorn=[] # yes  or no near a harbor
  for k in range(len(df1)): # get positions away from the dock
        #[la,lo]=dd2dm(list(df1[2])[k],list(df1[3])[k])# converts to ddmm format
        [la,lo]=dd2dm(list(df1[8])[k],list(df1[7])[k])# converts to ddmm format change indexes 4/9/2019
        yorn.append(gps_compare_JiM(la,lo,mindistfromharbor))
  df1[4]=yorn # this is a list of yes and no's
  df1=df1[df1[4]=='no'] # forms a new dataframe with only those positions away from the dock
  ####
  if len(df1)>0:# note in some cases the vessel may have only reported temps from the dock
    [vn,sn,form]=getvesselname(ves[j])#gets vessel name, probe serial number, and fixed_or_mobile for this vessel number
    print(vn)
    last_wifi_date=getlastwifi(sn)# gets date of last wifi for this serieal number
    last_fishing_date=getlastfished(vn,mindistfromharbor)
    obst=str(round(c2f(float(df1[14][-1:].values[0]))[0],1)) # last observation for this vessel
    #climt=str(round(c2f(getclim(df1[8][-1:].values[0],df1[7][-1:].values[0],str(int(max(df1.datet).to_pydatetime().strftime('%j')))))[0],1))# climatology for this day based on Melrose
    climt='0.0'
    print('looking for NGDC depth')
    depth_ngdc=get_depth(df1[7][-1:].values[0],df1[8][-1:].values[0],mindist_allowed)# ngdc bottom depth estimate
    if abs(df1[11][-1:].values[0]-depth_ngdc)/depth_ngdc>0.15:# big difference in observed and NGDC depth
      obst=obst+'*'
    #get FVCOM
    print('looking for FVCOM')
    try:
        # JiM modified the "str(round(" prefix on 7/8/2020
        modt=str(round(c2f(get_FVCOM_bottom_temp(df1[8][-1:].values[0],df1[7][-1:].values[0],max(df1.datet).to_pydatetime(),-1))[0],1))
        if abs(float(obst)-float(modt))>maxdiff: #something must be wrong
          modt='nan'
    except:
        print('FVCOM is down?')
        modt='nan' # the FVCOM must be down
    # get DOPPIO
    if max(df1.datet).to_pydatetime()>=dt(2017,11,1,0,0,0):
      print('looking for DOPPIO')
      try:
        modt_doppio=str(round(c2f(doppio_modules.get_doppio(lat=df1[8][-1:].values[0],lon=df1[7][-1:].values[0],depth='bottom',time=max(df1.datet).to_pydatetime()-timedelta(days=4),fortype='temperature'))[0],1))
      except:
        modt_doppio=np.nan
      print('looking for GOMOFS')
      try:
        modt_gomofs=str(round(c2f(gomofs_modules.get_gomofs(max(df1.datet).to_pydatetime()-timedelta(days=0),df1[8][-1:].values[0],df1[7][-1:].values[0],'bottom'))[0],1))
      except:
        modt_gomofs=np.nan
    if (last_fishing_date!='unknown') and (last_fishing_date!='retired'):
      if form=='f':
        days_to_worry=10 # fixed gear case because they do not always haul the trap each trip
      else:
        days_to_worry=1 # mobile case
      if max(df1.datet).to_pydatetime()<parser.parse(last_fishing_date)-timedelta(days=days_to_worry): # makes sure there are at least two days difference
         add_color=' class=redtext'
      else:
         add_color=''
    else:
      add_color=''
    outfile.write('
'+vn+'	'+''+max(df1.datet).strftime("%Y-%m-%d")+'	'+obst+'	'+modt+'	'+str(modt_doppio)+'	'+str(modt_gomofs)+'	'+climt+'	'+str(len(df1))+'	'+sn+'	'+last_wifi_date+'	'+last_fishing_date+'
\n')
    #outfile.write('
'+vn+'	'+''+max(df1.datet).strftime("%b %d,%Y")+'	'+obst+'	'+modt+'	'+modt_doppio+'	coming soon	'+climt+'	'+str(len(df1))+'	'+sn+'	'+last_wifi_date+'	'+last_fishing_date+'
\n')

    sumnum=sumnum+len(df1)
    if ~np.isnan(float(climt)):
       anomaly.append(float(obst)-float(climt))
    if ~np.isnan(float(modt)):
       moderror.append(float(obst)-float(modt))
    if ~np.isnan(float(modt_doppio)):
       moderror_doppio.append(float(obst)-float(modt_doppio))
    if ~np.isnan(float(modt_gomofs)):
       moderror_gomofs.append(float(obst)-float(modt_gomofs))

outfile.write('
TOTAL							'+str(sumnum)+'
\n')
outfile.write('

')
outfile.write('

 * indicates maximum observed depth is more than 15% different from what the bottom depth is at that location (according to NGDC)')
outfile.write('

 Most recent mean observed minus Melrose-30-year-climatology = '+str(round(np.mean(anomaly),1))+' degF')
if str(round(np.mean(moderror),1))=='nan':
  outfile.write('

 Note: FVCOM model is evidently down.')
else:
  outfile.write('

 Most recent mean observed minus FVCOM GOM3 forecast model = '+str(round(np.mean(moderror),1))+' degF')
outfile.write('

 Most recent mean observed minus DOPPIO forecast model = '+str(round(np.mean(moderror_doppio),1))+' degF')
outfile.write('

 Most recent mean observed minus GOMOFS forecast model = '+str(round(np.mean(moderror_gomofs),1))+' degF')
outfile.write('

 Note: Vessels highlighted in red may require some attention.')
outfile.write('

 Note: To see a map of the last month of observations, click here.')
outfile.close()
#os.system('cp /net/pubweb_html/drifter/lastfix_temp.html /net/pubweb_html/drifter/lastfix.html')# changed from "mv" to "cp" on 4/9/2019
#os.system('cp /net/newfish_www/html/nefsc/emolt/lastfix_temp.html /net/newfish_www/html/nefsc/emolt/lastfix.html')# changed from "mv" to "cp" on 4/9/2019


from pylons import config

from ckan import plugins as p
try:
    from ckan.lib.plugins import DefaultTranslation
except ImportError:
    class DefaultTranslation():
        pass

from ckanext.iaest.logic import (iaest_dataset_show,
                                iaest_catalog_show,
                                iaest_catalog_search,
                                iaest_datasets_list,
                                iaest_auth,
                                iaest_federador,
                                )
from ckanext.iaest import utils

DEFAULT_CATALOG_ENDPOINT = '/catalog/iaest.{_format}'
CUSTOM_ENDPOINT_CONFIG = 'ckanext.iaest.catalog_endpoint'
ENABLE_CONTENT_NEGOTIATION_CONFIG = 'ckanext.iaest.enable_content_negotiation'

class IAESTPlugin(p.SingletonPlugin, DefaultTranslation):

    p.implements(p.IConfigurer, inherit=True)
    p.implements(p.IRoutes, inherit=True)
    p.implements(p.IActions, inherit=True)
    p.implements(p.IAuthFunctions, inherit=True)
    p.implements(p.IPackageController, inherit=True)
    if p.toolkit.check_ckan_version(min_version='2.5.0'):
        p.implements(p.ITranslation, inherit=True)

    # IConfigurer
    def update_config(self, config):
        p.toolkit.add_template_directory(config, 'templates')

        # Check catalog URI on startup to emit a warning if necessary
        utils.catalog_uri()

        # Check custom catalog endpoint
        custom_endpoint = config.get(CUSTOM_ENDPOINT_CONFIG)
        if custom_endpoint:
            if not custom_endpoint[:1] == '/':
                raise Exception(
                    '"{0}" should start with a backslash (/)'.format(
                        CUSTOM_ENDPOINT_CONFIG))
            if '{_format}' not in custom_endpoint:
                raise Exception(
                    '"{0}" should contain {{_format}}'.format(
                        CUSTOM_ENDPOINT_CONFIG))

    # IRoutes
    def before_map(self, _map):

        controller = 'ckanext.iaest.controllers:DCATController'

        _map.connect('iaest_catalog',
                     config.get('ckanext.iaest.catalog_endpoint',
                                DEFAULT_CATALOG_ENDPOINT),
                     controller=controller, action='read_catalog',
                     requirements={'_format': 'xml|rdf|n3|ttl|jsonld'})

        _map.connect('iaest_dataset', '/dataset/iaest/{_id}.{_format}',
                     controller=controller, action='read_dataset',
                     requirements={'_format': 'xml|rdf|n3|ttl|jsonld'})

        _map.connect('federador_rdf', '/federador.rdf',
                     controller=controller, action='federador')

        return _map

    # IActions
    def get_actions(self):
        return {
            'iaest_dataset_show': iaest_dataset_show,
            'iaest_catalog_show':iaest_catalog_show,
            'iaest_catalog_search': iaest_catalog_search,
            'iaest_federador':iaest_federador,
        }

    # IAuthFunctions
    def get_auth_functions(self):
        return {
            'iaest_dataset_show': iaest_auth,
            'iaest_catalog_show': iaest_auth,
            'iaest_catalog_search': iaest_auth,
            'iaest_federador': iaest_auth
        }

    # IPackageController
    def after_show(self, context, data_dict):

        if context.get('for_view'):
            field_labels = utils.field_labels()

            def set_titles(object_dict):
                for key, value in object_dict.iteritems():
                    if key in field_labels:
                        object_dict[field_labels[key]] = object_dict[key]
                        del object_dict[key]

            for resource in data_dict.get('resources', []):
                set_titles(resource)

            for extra in data_dict.get('extras', []):
                if extra['key'] in field_labels:
                    extra['key'] = field_labels[extra['key']]

        return data_dict

class DCATJSONInterface(p.SingletonPlugin):

    p.implements(p.IRoutes, inherit=True)
    p.implements(p.IActions)
    p.implements(p.IAuthFunctions, inherit=True)

    # IRoutes
    def after_map(self, map):

        controller = 'ckanext.iaest.controllers:DCATController'
        route = config.get('ckanext.iaest.json_endpoint', '/dcat.json')
        map.connect(route, controller=controller, action='dcat_json')

        return map

    # IActions
    def get_actions(self):
        return {
            'iaest_datasets_list': iaest_datasets_list,
        }

    # IAuthFunctions
    def get_auth_functions(self):
        return {
            'iaest_datasets_list': iaest_auth,
        }