import argparse
from collections import defaultdict

def get_gold_dict(filename):
  gold = defaultdict(list)
  with open(filename) as f:
    for line in f:
      line = line.strip()
      w1, w2 = line.split(" ")
      #w2 = w2.lower()
      gold[w1].append(w2)
  return gold

def get_gold_bins(filename):
  gold = [defaultdict(list) for i in range(5)]
  count = 0
  with open(filename) as f:
    for line in f:
      line = line.strip()
      w1, w2 = line.split(" ")
      #w2 = w2.lower()
      _gold = gold[count // 300]
      if w1 not in _gold: count += 1
      _gold[w1].append(w2)
  return gold

def get_D(filename):
  D = {}
  with open(filename) as f:
    for line in f:
      line = line.strip()
      w1, w2 = line.split("\t")
      if w2 == "EMPTY": continue
      D[w1] = w2.lower()
  return D

def get_DD(filename):
  D = {}
  with open(filename) as f:
    for line in f:
      line = line.strip()
      w1, w2 = line.split("\t")
      if w2 == "EMPTY": continue
      w1, w2 = w1.lower(), w2.lower()
      if w1 not in D: D[w1] = w2
  return D

def get_prec(D, DD, gold):
  count = 0
  right = 0
  unk = []
  for word in gold:
    if word not in D:
      if word not in DD:
        unk.append(gold[word][0])
        if word in gold[word]: right += 1
        #count -= 1
      else:
        if DD[word] in gold[word]: right += 1
    else:
      if D[word] in gold[word]: right += 1
    count += 1
  #print(unk)
  #print(len(unk))
  #print(count)
  return 1.0 * right / count

def main(args):
  gold = get_gold_dict(args.gold)
  D = get_D(args.D)
  DD = get_DD(args.D)
  print(get_prec(D, DD, gold))
  gb = get_gold_bins(args.gold)
  for g in gb:
    print(get_prec(D, DD, g))

if __name__ == "__main__":
  parser = argparse.ArgumentParser()
  res_dir = "/data00/home/chizewen/res/en2it"
  parser.add_argument("--D", default="./out/80Kgreedy.it")
  parser.add_argument("--gold",
    default="/Users/chizewen/repo/transmat/data/OPUS_en_it_europarl_test.txt")
  args = parser.parse_args()
  main(args)


import torch
import pdb

class FPNtrainer :
	def __init__(self, args, logger, data, model):
		self.args = args
		self.name = args.name
		self.experiment_name = args.save_name
		self.device = args.device

		# trainee
		self.model = model

		# constants
		self.epochs = args.epochs

		# datasets
		self.data = data
		self.tr_loader = data.train_loader
		self.vl_loader = data.valid_loader
		self.te_loader = data.test_loader

	def train(self):

		for epoch in range(1, self.epochs+1):
			for b_idx, (images, boxes, labels) in enumerate(self.tr_loader):
				images = images.to(self.device)
				boxes = [b.to(self.device) for b in boxes]
				labels = [l.to(self.device) for l in labels]
				pdb.set_trace()

        # images = datas[0]
        # boxes = datas[1]
        # labels = datas[2]

        # images = images.to(device)
        # boxes = [b.to(device) for b in boxes]
        # labels = [l.to(device) for l in labels]

        # pred = model(images)
        # loss, (loc, cls) = criterion(pred, boxes, labels)

        # # sgd
        # optimizer.zero_grad()
        # loss.backward()
        # optimizer.step()

        # toc = time.time()


"""Easee charger switch."""

from homeassistant.components.switch import SwitchEntity

from .entity import ChargerEntity
from .const import DOMAIN
import logging

_LOGGER = logging.getLogger(__name__)

async def async_setup_entry(hass, entry, async_add_entities):
    """Setup switch platform."""
    controller = hass.data[DOMAIN]["controller"]
    entities = controller.get_switch_entities()
    async_add_entities(entities)

class ChargerSwitch(ChargerEntity, SwitchEntity):
    """Easee switch class."""

    async def async_turn_on(self, **kwargs):  # pylint: disable=unused-argument
        """Turn on the switch."""
        _LOGGER.debug("%s Switch turn on" % self._entity_name)
        self._state = True
        self.async_write_ha_state()
        function_call = getattr(self.charger_data.charger, self._switch_func)
        await function_call(True)

    async def async_turn_off(self, **kwargs):  # pylint: disable=unused-argument
        """Turn off the switch."""
        _LOGGER.debug("%s Switch turn off" % self._entity_name)
        self._state = False
        self.async_write_ha_state()
        function_call = getattr(self.charger_data.charger, self._switch_func)
        await function_call(False)

    @property
    def is_on(self):
        """Return true if the switch is on."""
        _LOGGER.debug("Getting state of %s" % self._entity_name)
        return self._state


from sklearn.model_selection import train_test_split
import pandas as pd
from keras_recommender.library.cf import CollaborativeFilteringV1

def main():
    data_dir_path = './data/ml-latest-small'
    output_dir_path = './models'

    records = pd.read_csv(data_dir_path + '/ratings.csv')
    print(records.describe())

    ratings_train, ratings_test = train_test_split(records, test_size=0.2, random_state=0)

    user_id_train = ratings_train['userId']
    item_id_train = ratings_train['movieId']
    rating_train = ratings_train['rating']

    user_id_test = ratings_test['userId']
    item_id_test = ratings_test['movieId']
    rating_test = ratings_test['rating']

    max_user_id = records['userId'].max()
    max_item_id = records['movieId'].max()

    config = dict()
    config['max_user_id'] = max_user_id
    config['max_item_id'] = max_item_id

    cf = CollaborativeFilteringV1()
    history = cf.fit(config=config, user_id_train=user_id_train,
                     item_id_train=item_id_train,
                     rating_train=rating_train,
                     model_dir_path=output_dir_path)

    metrics = cf.evaluate(user_id_test=user_id_test,
                          item_id_test=item_id_test,
                          rating_test=rating_test)

if __name__ == '__main__':
    main()


from __future__ import print_function
import argparse
import os
import pdb
import random
import torch
import torch.nn as nn
import torch.nn.parallel
import torch.backends.cudnn as cudnn
import torch.optim as optim
import torch.utils.data
import torchvision.datasets as dset
import torchvision.transforms as transforms
import torchvision.utils as vutils
from tensorboardX import SummaryWriter
from MyDataset import MyDataset
from Generator import Generator
from Discriminator import Discriminator

parser = argparse.ArgumentParser()
parser.add_argument('--ngpu',           type=int, default=1, help='number of GPUs to use')
parser.add_argument('--dim_z',          type=int, default=100, help='size of the latent z vector')
parser.add_argument('--dim_g',          type=int, default=64)
parser.add_argument('--dim_d',          type=int, default=64)
parser.add_argument('--batch_size',     type=int, default=64, help='input batch size')
parser.add_argument('--nepoch',         type=int, default=25, help='number of epochs to train for')
parser.add_argument('--nworkers',       type=int, help='number of data loading workers', default=2)
parser.add_argument('--manual_seed',    type=int, help='manual seed')
parser.add_argument('--image_size',     type=int, default=64, help='the height / width of the input image to network')
parser.add_argument('--cuda',           action='store_true', help='enables cuda')
parser.add_argument('--lr',             type=float, default=0.0002, help='learning rate, default=0.0002')
parser.add_argument('--beta1',          type=float, default=0.5, help='beta1 for adam. default=0.5')
# parser.add_argument('--dataset',        required=True, help='cifar10 | lsun | mnist |imagenet | folder | lfw | fake')
parser.add_argument('--dataroot',       required=True, help='path to dataset')
parser.add_argument('--output_folder',  default='.', help='folder to output images and model checkpoints')
parser.add_argument('--log_directory',  type=str, help='directory to save log summaries', default="./log/1225/")
parser.add_argument('--netG',           default='', help="path to netG (to continue training)")
parser.add_argument('--netD',           default='', help="path to netD (to continue training)")

def weights_init(m):
    classname = m.__class__.__name__
    if classname.find('Conv') != -1:
        m.weight.data.normal_(0.0, 0.02)
    elif classname.find("BatchNorm") != -1:
        m.weight.data.normal_(1.0, 0.02)
        m.bias.data.fill_(0)

if __name__ == '__main__':
    opt = parser.parse_args()
    print(opt)

    try:
        os.makedirs(opt.output_folder)
    except OSError:
        pass

    if opt.manual_seed is None:
        opt.manual_seed = random.randint(1, 100000)
    print("Random Seed: ", opt.manual_seed)
    random.seed(opt.manual_seed)
    torch.manual_seed(opt.manual_seed)

    cudnn.benchmark = True

    if torch.cuda.is_available() and not opt.cuda:
        print('WARNING: You have a CUDA device, so you should probably run with --cuda')

    train_augmentation = transforms.Compose([transforms.Resize(opt.image_size),
                                             transforms.CenterCrop(opt.image_size),
                                             transforms.ToTensor(),
                                             transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5))])
    dataset = MyDataset(opt.dataroot, train_augmentation)
    num_channel = 3

    # if opt.dataset in ['imagenet', 'folder', 'lfw']:
    #     dataset = dset.ImageFolder(root=opt.dataroot,
    #                             transform=transforms.Compose([
    #                                 transforms.Resize(opt.image_size),
    #                                 transforms.CenterCrop(opt.image_size),
    #                                 transforms.ToTensor(),
    #                                 transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5)), ##
    #                             ]))
    #     num_channel = 3
    # elif opt.dataset == 'lsun':
    #     dataset = dset.LSUN(root=opt.dataroot, classes=['bedroom_train'],
    #                         transform=transforms.Compose([
    #                             transforms.Resize(opt.image_size),
    #                             transforms.CenterCrop(opt.image_size),
    #                             transforms.ToTensor(),
    #                             transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5)),
    #                         ]))
    #     num_channel = 3
    # elif opt.dataset == 'cifar10':
    #     dataset = dset.CIFAR10(root=opt.dataroot, download=True,
    #                         transform=transforms.Compose([
    #                             transforms.Resize(opt.image_size),
    #                             transforms.ToTensor(),
    #                             transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5)),
    #                         ]))
    #     num_channel = 3
    # elif opt.dataset == 'mnist':
    #     dataset = dset.MNIST(root=opt.dataroot, download=True,
    #                         transform=transforms.Compose([
    #                             transforms.Resize(opt.image_size),
    #                             transforms.ToTensor(),
    #                             transforms.Normalize((0.5,), (0.5,)),
    #                         ]))
    #     num_channel = 1
    # elif opt.dataset == 'fake':
    #     dataset = dset.FakeData(image_size=(3, opt.imageSize, opt.imageSize),
    #                             transform=transforms.ToTensor())
    #     num_channel = 3
    assert dataset
    dataloader = torch.utils.data.DataLoader(dataset, batch_size=opt.batch_size,
                                            shuffle=True, num_workers=int(opt.nworkers))

    device = torch.device("cuda:0" if opt.cuda else "cpu")
    ngpu  = int(opt.ngpu)
    dim_z = int(opt.dim_z)
    dim_g = int(opt.dim_g)
    dim_d = int(opt.dim_d)

    netG = Generator(ngpu, dim_z, dim_g, num_channel).to(device)
    netG.apply(weights_init)
    if opt.netG != '':
        netG.load_state_dict(torch.load(opt.netG))
    print(netG)

    netD = Discriminator(ngpu, dim_d, num_channel).to(device)
    netD.apply(weights_init)
    if opt.netD != '':
        netD.load_state_dict(torch.load(opt.netD))
    print(netD)

    criterion = nn.BCELoss()

    fixed_noise = torch.randn(opt.batch_size, dim_z, 1, 1, device=device)
    real_label = 1
    fake_label = 0

    optimD = optim.Adam(netD.parameters(), lr=opt.lr, betas=(opt.beta1, 0.999))
    optimG = optim.Adam(netG.parameters(), lr=opt.lr, betas=(opt.beta1, 0.999))

    writer = SummaryWriter(log_dir=opt.log_directory)

    for epoch in range(opt.batch_size):
        for i, data in enumerate(dataloader):
            # (1) Update D network: minimize -( log(D(x)) + log(1 - D(G(z))) )
            netD.zero_grad()
            real = data.to(device)  ## [0]?
            batch_size = real.size(0)
            label = torch.full((batch_size,), real_label, device=device)

            output = netD(real)
            errD_real = criterion(output, label)
            errD_real.backward()
            D_x = output.mean().item()

            noise = torch.randn(batch_size, dim_z, 1, 1, device=device)
            fake = netG(noise)
            label.fill_(fake_label)
            output = netD(fake.detach())
            errD_fake = criterion(output, label)
            errD_fake.backward()
            D_G_z1 = output.mean().item()
            errD = errD_real + errD_fake
            optimD.step()

            # (2) Update G network: minimize -log(D(G(z)))
            netG.zero_grad()
            label.fill_(real_label)
            output = netD(fake)
            errG = criterion(output, label)
            errG.backward()
            D_G_z2 = output.mean().item()
            optimG.step()

            print('[%d/%d][%d/%d] LossD: %.4f, LossG: %.4f, D(x): %.4f, D(G(z)): %.4f & %.4f' \
            % (epoch, opt.nepoch, i, len(dataloader), errD.item(), errG.item(), D_x, D_G_z1, D_G_z2))

            writer.add_scalar('Generator/Loss',     errG.item(), len(dataloader) * (epoch - 1) + i)
            writer.add_scalar('Discriminator/Loss', errD.item(), len(dataloader) * (epoch - 1) + i)

            if i % 100 == 0:
                vutils.save_image(real, '%s/real_samples.png' % opt.output_folder, normalize=True)
                fake = netG(fixed_noise)
                vutils.save_image(fake.detach(), '%s/fake_samples_epoch_%03d.png' % (opt.output_folder, epoch), normalize=True)

        # do checkpointing
        torch.save(netG.state_dict(), '%s/netG_epoch_%d.pkl' % (opt.output_folder, epoch))
        torch.save(netD.state_dict(), '%s/netD_epoch_%d.pkl' % (opt.output_folder, epoch))


#!/usr/bin/env python3
#! python3

import re
import time
import asyncio
from constants import *
from youtube_dl import YoutubeDL
from apricot.utils import parse_qs, urlparse

class YoutubeExtractor(object):

	def __init__(self, session):
		self.session = session
		self.ytdl	= YoutubeDL(YTDL_OPTS)

	def getVideoID(self, url):
		""" Extract the video id from a url, return video id as str. """
		idregx = re.compile(r'[\w-]{11}$')
		url = str(url)

		if idregx.match(url):
			return url # ID of video

		# extract info
		if '://' not in url:
			url = '//' + url
		parsedurl = urlparse(url)
		if parsedurl.netloc in ('youtube.com', 'www.youtube.com'):
			query = parse_qs(parsedurl.query)
			if 'v' in query and idregx.match(query['v'][0]):
				return query['v'][0]
		elif parsedurl.netloc in ('youtu.be', 'www.youtu.be'):
			vidid = parsedurl.path.split('/')[-1] if parsedurl.path else ''
			if idregx.match(vidid):
				return vidid

		# raise error if not found
		err = "Need 11 character video id or the URL of the video. Got %s"
		raise ValueError(err % url)

	def timeToSeconds(self, t):
		""" Convert time_dict to seconds """
		return t['seconds'] + (t['minutes'] * 60) + (t['hours'] * 60 * 60)

	def timeToText(self, _time):
		""" Convert time_dict to formatted text """
		secs   = self.timeToSeconds(_time)
		FORMAT = '%M:%S'
		if _time['hours'] > 0: FORMAT = '%H:' + FORMAT
		return time.strftime(FORMAT, time.gmtime(secs))

	async def getShorten(self, url):
		""" Use Google URL Shortener to shorten url """
		query   = GOOGLE_SHORT.format(key=GOOGLE_API_KEY)
		headers = {'content-type': 'application/json'}
		payload = {"longUrl": url}

		resp    = await self.session.post(query, headers=headers, json=payload)
		data    = await resp.json()
		return data['id']

	async def getThumbnail(self, url):
		""" Get the thumbnail url of a video """
		url   = self.getVideoID(url)
		thumb = MAX_THUMBNAIL.format(id=url)

		# see if maxres thumbnail is valid
		resp  = await self.session.head(thumb)

		# if not, fall back to default hq thumbnail
		if str(resp.status).startswith('4'):
			thumb = MIN_THUMBNAIL.format(id=url)

		return thumb

	async def getTitle(self, url):
		""" Get the title of a youtube url """
		link_copy   = str(YOUTUBE_LINK)
		oembed_copy = str(YOUTUBE_OEMBED)
		url  = self.getVideoID(url)
		url  = link_copy.format(id=url)
		url  = oembed_copy.format(url=url)

		resp = await self.session.get(url)
		data = await resp.json()

		return data['title']

	async def getAudioStream(self, url, future=None):
		""" Get the M4A audio stream url of a video """
		url  = self.getVideoID(url)
		info = self.ytdl.extract_info(url, download=False)

		# get audio url
		audio_info = [
			f for f in info['formats']
			if 'audio' in f['format'].lower()
		]
		audio_info = audio_info[0] # get first link

		# set future if any and return result
		if future is not None:
			try: future.set_result(audio_info['url'])
			except: pass
		return audio_info['url']

	async def getVideoTime(self, url):
		""" Get video duration in {"hours:int", "minutes":int, "seconds":int, "stream":bool} """
		gdata_copy = str(YOUTUBE_GDATA)
		url = self.getVideoID(url)
		url = gdata_copy.format(id=url, key=GOOGLE_API_KEY)

		# extract duration and create time_dict
		resp = await self.session.get(url)
		info = await resp.json()
		info = info['items'][0]['contentDetails']
		duration = str(info['duration'])
		duration = duration[2:].lower() # remove "PT"
		TIME = {'hours':0,'minutes':0,'seconds':0}

		# populate time_dict with values
		for unit in TIME:
			if unit[0] in duration:
				valid = duration.split(unit[0])[0]
				valid = re.sub('\D', '', valid)
				TIME[unit] = int(valid[-2:])
		if max([TIME[t] for t in TIME]) < 1:
			TIME['stream'] = True
		else:
			TIME['stream'] = False

		# return time_dict
		return TIME

	async def getSearchResults(self, search, results=1):
		""" return array of {'id':str, 'title':str} from youtube search """
		params = {
			"key": GOOGLE_API_KEY,
			"q": search,
			"maxResults": str(results),
			"part": "snippet"
		}

		# fetch json info
		resp = await self.session.get(YOUTUBE_SEARCH, params=params)
		data = await resp.json()

		# get results
		_results_ = []
		_found_   = 0
		for item in data['items']:
			video_id    = item['id']['videoId']
			video_title = item['snippet']['title']
			info = {'id': video_id, 'title': video_title}
			_found_ += 1
			if _found_ <= results:
				_results_.append(info)
			if _found_ == results:
				break

		return _results_

	async def getInfo(self, url, search=False):
		""" Get all info of video from id """

		# get first url from search if enabled
		if search:
			info = await self.getSearchResults(url, results=1)
			url = info[0]['id']
		url   = self.getVideoID(url)

		# spawn audio fetcher in a non blocking task
		# so that other operations can finish first
		def on_audio_complete(task):
			task.event.set()
		future     = asyncio.Future()
		audio_coro = lambda: self.getAudioStream(url, future)
		task       = self.session.loop.create_task(audio_coro())
		task.event = asyncio.Event()
		task.add_done_callback(on_audio_complete)

		# get info
		_thumb = await self.getThumbnail(url)
		_title = await self.getTitle(url)
		_time  = await self.getVideoTime(url)

		# wait for audio fetching to complete
		# and get result
		await task.event.wait()
		_audio = future.result()

		# create dict
		info = {}
		info['thumbnail'] = _thumb
		info['title']     = _title
		info['audio']     = _audio
		info['time']      = _time

		# return final built info
		return info


# -*- coding: utf-8 -*-
"""
Reference implementation of RiWalk.
Author: Xuewei Ma
For more details, refer to the paper:
RiWalk: Fast Structural Node Embedding via Role Identification
ICDM, 2019
"""

import argparse
from . import RiWalkGraph
from .utils import WALK_FILES_DIR
from gensim.models import Word2Vec
from gensim.models.keyedvectors import Word2VecKeyedVectors
import networkx as nx
import os
import glob
from semb.methods import BaseMethod

class Sentences(object):
    def __init__(self, file_names):
        self.file_names = file_names

    def __iter__(self):
        fs = []
        for file_name in self.file_names:
            fs.append(open(file_name))
        while True:
            flag = 0
            for i, f in enumerate(fs):
                line = f.readline()
                if line != '':
                    flag = 1
                    yield line.split()
            if not flag:
                try:
                    for f in fs:
                        f.close()
                except:
                    pass
                return

class Method(BaseMethod):

    __PARAMS__ = dict(dim=128, walk_length=10, num_walks=80, window_size=10,
                      until_k=4, iter=5, workers=5, flag='wl')

    def get_id(self):
        return "riwalk"

    def train(self):
        # remove existing files
        os.system('rm -rf %s' %
                  os.path.join(WALK_FILES_DIR, "__random_walks_*.txt"))
        nx_g, mapping = self.preprocess_graph(self.graph)
        reps = self.learn(nx_g, mapping)
        self.embeddings = dict()
        for cur_node in self.graph.nodes():
            self.embeddings[cur_node] = reps.get_vector(str(cur_node)).tolist()

    def learn_embeddings(self):
        """
        learn embeddings from random walks.
        hs:  0:negative sampling 1:hierarchica softmax
        sg:  0:CBOW              1:skip-gram
        """
        dim = self.params['dim']
        window_size = self.params['window_size']
        workers = self.params['workers']
        iter_num = self.params['iter']

        walk_files = glob.glob(os.path.join(
            WALK_FILES_DIR, "__random_walks_*.txt"))
        sentences = Sentences(walk_files)
        model = Word2Vec(sentences, size=dim, window=window_size,
                         min_count=0, sg=1, workers=workers, iter=iter_num)
        return model.wv

    def preprocess_graph(self, nx_g):
        """
        1. relabel nodes with 0,1,2,3,...,N.
        2. convert graph to adjacency representation as a list of tuples.
        """
        mapping = {_: i for i, _ in enumerate(nx_g.nodes())}
        nx_g = nx.relabel_nodes(nx_g, mapping)
        nx_g = [tuple(nx_g.neighbors(_)) for _ in range(len(nx_g))]

        return nx_g, mapping

    def learn(self, nx_g, mapping):
        g = RiWalkGraph.RiGraph(nx_g, self.params)
        g.process_random_walks()

        wv = self.learn_embeddings()

        original_wv = Word2VecKeyedVectors(self.params['dim'])
        original_nodes = list(mapping.keys())
        original_vecs = [wv.word_vec(str(mapping[node]))
                         for node in original_nodes]
        original_wv.add(entities=list(map(str, original_nodes)),
                        weights=original_vecs)
        return original_wv


# -*- coding: utf-8 -*-
import numpy as np
import time
import matplotlib.pyplot as plt
from scipy.spatial.distance import pdist
from scipy.spatial.distance import squareform
from mpl_toolkits.mplot3d import Axes3D
# 绘制不同的点类别
def plotFeature(data, clusterArr):
    clusterNum = len(clusterArr)
    ax = plt.figure().add_subplot(111, projection='3d')
    scatterColors = ['red', 'blue', 'orange', 'orchid', 'black', '#E0E3E6', '#E0E0E6', '#E5E0E6', '#E6E0E4', '#E6E0E4',
                     'brown']
    for i in range(clusterNum):
        colorSytle = scatterColors[i % len(scatterColors)]
        subCluster = data[clusterArr[i].indexSet]
        ax.scatter(subCluster[:, 0], subCluster[:, 1], subCluster[:, 2], c=colorSytle, s=20)
    plt.show()

# 凝聚层次聚类算法的簇集
class AGNESCluster(object):
    def __init__(self, center, startId, endId, indexSet):
        # 中心坐标
        self.center = center
        # 起索引（start）
        self.startId = startId
        # 始索引（end）
        self.endId = endId
        # 该簇包含的点索引集合
        self.indexSet = indexSet
    def __repr__(self):
        return "------------\ncenter:%s\nstartId:%s\nendId:%s\nindedxSet:%s\n"%(self.center,self.startId,self.endId,self.indexSet)
# 计算两个簇集的时间距离
def computerTime(cluster1,cluster2,timeArr):
    # 如果两个簇集相离，时间距离为计算
    if(cluster1.endId<cluster2.startId):
        return timeArr[cluster2.startId]-timeArr[cluster1.endId]
    elif(cluster1.startId > cluster2.endId):
        return timeArr[cluster1.startId] - timeArr[cluster2.endId]
    # 如果两个首尾都相等，那就是一个簇集
    elif (cluster1.startId == cluster2.startId) and (cluster1.endId == cluster2.endId):
        return 0
    else:
    # 如果两个簇集相交，时间距离为-1
        return -1
# 更新簇集更新矩阵
# clusterArr：簇集
# clusterDisMat：当前的簇集空间距离矩阵
# clusterTimeMat：当前的簇集时间距离矩阵
# timeArr：每一个点的时间
# needHBId：需要被替换的距离数组索引
# deleteClusterId：需要被删除的簇集索引
def updateClusterMat(clusterArr,clusterDisMat,clusterTimeMat, timeArr,needHBId,deleteClusterId):
    # 一共有n个簇集
    n = len(clusterArr)
    # 删除第deleteClusterId空间距离，主要为第deleteClusterId行和deleteClusterId列
    clusterDisMat = np.delete(clusterDisMat, deleteClusterId, axis=1)
    clusterDisMat = np.delete(clusterDisMat, deleteClusterId, axis=0)
    # 删除第deleteClusterId时间距离，主要为第deleteClusterId行和deleteClusterId列
    clusterTimeMat = np.delete(clusterTimeMat, deleteClusterId, axis=1)
    clusterTimeMat = np.delete(clusterTimeMat, deleteClusterId, axis=0)
    # 需要被更新的时间距离与空间距离
    upDisArr = []
    upTimeArr = []
    for i in range(n):
        d = np.linalg.norm(clusterArr[i].center - clusterArr[needHBId].center)
        upDisArr.append(d)
        upTimeArr.append(computerTime(clusterArr[i], clusterArr[needHBId], timeArr))
    # 更新时间距离与空间距离
    clusterDisMat[needHBId] = upDisArr
    clusterDisMat[:, needHBId] = upDisArr
    clusterTimeMat[needHBId] = upTimeArr
    clusterTimeMat[:, needHBId] = upTimeArr
    return clusterDisMat, clusterTimeMat
# 簇集合并函数
def getNewCluster(cluster1, cluster2, xyArr):
    # 第一个簇集包含的点集
    list1 = cluster1.indexSet
    # 第二个簇集包含的点集
    list2 = cluster2.indexSet
    # 新簇集所包含的点集,注意：两个簇集中不可能包含重复的点集，所以不考虑去重的现象
    newList = np.concatenate([list1,list2])
    # 计算新簇集中心点的坐标
    newCenter = np.mean(np.concatenate([xyArr[list1],xyArr[list2]]),axis=0)
    # 计算新簇集的起点
    newStartId = np.min(newList)
    # 计算新簇集的终点
    newEndId = np.max(newList)
    # 返回新簇集
    return AGNESCluster(newCenter,newStartId,newEndId,newList)
# 获得每一个簇集最近的簇集，以及最近簇集对应的距离
# clusterTimeMat簇集之间的时间距离矩阵
# clusterDisMat簇集之间的空间距离矩阵
# timeThreh簇集之间的时间阈值
# disThreh簇集之间的时间阈值
# clusterFloorId每个簇集对应的楼层
# 对比getClosestCluster的实现（与Indoor_STAGNES_DIS对比，此算法的实现慢一倍）
def getClosestCluster(clusterTimeMat,clusterDisMat,timeThreh,clusterFloorIds):
    # 返回的最近的簇集ID
    closedClusterId = []
    # 返回的最近的簇集距离
    closedClusterDis = []
    # 获得一共含有多少簇集
    n = clusterTimeMat.shape[0]
    for i in range(n):
        # 满足四个条件：时间阈值小于timeThreh（但是不能为0），空间阈值小于disThreh，楼层ID等于clusterFloorIds[i]
        # 如果满足条件，返回本身的距离值，否则返回np.inf（这里是一个技巧）
        disArr = np.where(((clusterTimeMat[i] < timeThreh) & (clusterTimeMat[i] != 0) & (clusterFloorIds == clusterFloorIds[i])), clusterDisMat[i],np.inf)
        closedClusterId.append(np.argmin(disArr))
        closedClusterDis.append(np.min(disArr))
    return np.array(closedClusterId), np.array(closedClusterDis)
# data 的第一列是unix时间戳，第二、三列是空间X,Y坐标，第四列是楼层ID
# disThreh 空间阈值
# timeThreh 时间阈值
def Indoor_STAGNES(data,classNum,timeThreh):
    # 获得数据的行和列(一共有n条数据)
    n, m = data.shape
    # 获得每一个轨迹点的时间数组
    timeArr=data[:,0]
    # 获得初始的簇集时间距离矩阵（此矩阵不断发生变化）
    clusterTimeMat = squareform(pdist(timeArr.reshape(-1, 1), metric='euclidean'))
    # 获得空间距离矩阵
    xyArr = data[:,1:3]
    # 获得初始的簇集空间距离矩阵（此矩阵不断发生变化）
    clusterDisMat = squareform(pdist(xyArr,metric='euclidean'))
    # 获得每一个簇集所在的楼层，Indoor的关键所在
    clusterFloorIds = data[:,3]
    # 将每一个点都初始化为一个簇集
    clusterArr = []
    for i in range(n):
        # 起初每个点都是一个簇，起始索引都是自己本身，且簇只包含自己本身
        clusterArr.append(AGNESCluster(xyArr[i], i, i, [i]))
    # 获得每一个簇集最近的"簇集"，以及相应的距离
    closedClusterId, closedClusterDis = getClosestCluster(clusterTimeMat,clusterDisMat,timeThreh,clusterFloorIds)
    while len(clusterArr) > classNum:
        # 获得需要合并的簇集ID
        needHBId = np.argmin(closedClusterDis)
        # 需要合并的两个簇
        currentCluster1 = clusterArr[needHBId]
        currentCluster2 = clusterArr[closedClusterId[needHBId]]
        # 合并后的新的簇集
        newCluster = getNewCluster(currentCluster1, currentCluster2, xyArr)
        # 赋予新的簇集
        clusterArr[needHBId] = newCluster
        # 删除被合并掉的簇集及所在的楼层
        del clusterArr[closedClusterId[needHBId]]
        clusterFloorIds = np.delete(clusterFloorIds, closedClusterId[needHBId])
        # 更新簇集距离矩阵
        clusterDisMat, clusterTimeMat = updateClusterMat(clusterArr, clusterDisMat, clusterTimeMat, timeArr, needHBId,
                                                         closedClusterId[needHBId])
        closedClusterId, closedClusterDis = getClosestCluster(clusterTimeMat, clusterDisMat, timeThreh,clusterFloorIds)
    return clusterArr
data = np.loadtxt("./data/cluster_unix_time_indoor.csv", delimiter=",")
start = time.clock()
clusterArr = Indoor_STAGNES(data,5,30)
end = time.clock()
print('finish all in %s' % str(end - start))
plotFeature(data[:,1:], clusterArr)


#!/usr/bin/python3
# -*- coding: utf-8 -*-

import scipy.io as scio
import sys
import os

class myConfig:
    """
    配置模块
    """
    # 构造函数
    def __init__(self):
        filePath = os.path.dirname(os.path.realpath(sys.argv[0])) + '//config'
        self.fileName = filePath + '//config.mat'
        if os.path.exists(filePath) is False:
            try:
                os.makedirs(filePath)
            except Exception as e:
                print(e)
        self.config = self.loadConfig()
        self.saveConfigFunction = self.saveConfig

    def setConfigToDefault(self):
        """
        默认配置
        """
        config = {
            'purchaseMode': 1,  # 开启进货界面
            'itemsPerPage': 10,  # 货物清单界面没页显示的商品数量
        }
        return config

    def saveConfig(self):
        """
        保存配置
        """
        scio.savemat(self.fileName, self.config)

    def loadConfig(self):
        """
        加载配置
        """
        if os.path.isfile(self.fileName) is False:
            config = self.setConfigToDefault()
            # self.saveConfig()
        else:
            config = scio.loadmat(self.fileName)
            config.pop('__header__')
            config.pop('__version__')
            config.pop('__globals__')
        return config


import json
import torch
import nltk
from InferSent.models import InferSent
import numpy as np

def get_infersent(V=2):
    '''
    Builds the infersent model using either GloVe or fastText
    '''
    MODEL_PATH = 'encoder/infersent%s.pkl' %V
    if V == 2:
        W2V_PATH = 'fastText/crawl-300d-2M.vec'
    elif V == 1:
        W2V_PATH = 'GloVe/glove.840B.300d.txt'

    params_model = {'bsize': 64, 'word_emb_dim': 300, 'enc_lstm_dim': 2048, \
                    'pool_type': 'max', 'dpout_model': 0.0, 'version': V}
    infersent = InferSent(params_model)
    infersent.load_state_dict(torch.load(MODEL_PATH))
    infersent.set_w2v_path(W2V_PATH)

    return infersent

def get_dataset(loc: str):
    '''
    Get the dataset from file location 'loc'
    '''
    with open(loc) as infile:
        dataset = json.load(infile)

    return dataset

def get_embedding(infersent, sentences: list):
    '''
    Use sentences to build a sentence embedding for each context using infersent.
    Returns a list of sentence embeddings
    '''

    print("Getting Sentence Embeddings for %d sentences", len(sentences))
    # outputs a numpy array with n vectors of dimension 4096
    context_embeddings = []
    for sentence in sentences:
        # sentence is actually a list of sentences for context_i
        embeddings = infersent.encode(sentence, tokenize=True)
        context_embeddings.append(embeddings)

    return np.asarray(context_embeddings)

def retrieve_data(dataset: dict):
    '''
    Retrieves context, questions, and targets from the data
    Context will return a list of lists for each sentence in a given context
    Questions will return a list of lists of questions for each context
    Targets will return a list of target values that correspond to each question.
    Target values are equivalent to the sentence number within the context that contains the answer to the question
    '''
    data = dataset['data']
    target = []
    ctx = []
    questions = []
    answers = []
    for topic in data:
        sentences = []
        for paragraph in topic['paragraphs']:
            context = paragraph['context']
            cont_sents = nltk.sent_tokenize(context)
            ctx.append(cont_sents)

            c_question = []
            c_answer = []
            for qas in paragraph['qas']:
                if qas['is_impossible']:
                    # skip impossible questions
                    continue
                question = qas['question']
                answer = qas['answers'][0]['text']
                c_question.append(question)
                c_answer.append(answer)

                ans_pos = qas['answers'][0]['answer_start']

                acc = 0
                # find which sentence the answer is part of
                for i, sent in enumerate(cont_sents):
                    acc += len(sent)
                    if acc > ans_pos:
                        # answer is in sentence i
                        target.append(i)
                        break

            questions.append(c_question)
            answers.append(c_answer)

    return ctx, questions, answers, target

def build_vocab(infersent, context: list):
    '''
    Flattens the context and then builds the vocab
    '''
    flat_context = [sentence for c in context for sentence in c]
    infersent.build_vocab(flat_context, tokenize=True)

    return infersent

def cos_similarity(a,b):
    '''
    Calculate the cosine similiarity between a and b
    cos_sim = a.b / |a||b|
    '''
    return np.dot(a,b) / (np.linalg.norm(a) * np.linalg.norm(b))

def main():
    n = 22
    infersent = get_infersent()
    print("Loading Dataset")
    dataset = get_dataset("train-v2.0.json")
    print("Parsing dataset")
    context, questions, answers, target = retrieve_data(dataset)
    print("Building vocab")
    build_vocab(infersent, context)
    print("Getting sentence embeddings")
    if n == -1:
        ctx_embed = get_embedding(infersent, context)
        q_embed = get_embedding(infersent, questions)
        a_embed = get_embedding(infersent, answers)
    else:
        ctx_embed = get_embedding(infersent, context[:n])
        q_embed = get_embedding(infersent, questions[:n])
        #a_embed = get_embedding(infersent, answers[:n])
    return ctx_embed, q_embed, a_embed


# Librerias
# ----------------------------------------------------------------------------------------------------
import time
import sys
import random
import requests # install requests
from bs4 import BeautifulSoup as bs # pip install beautifulsoup4
from datetime import datetime

# Functions
# ----------------------------------------------------------------------------------------------------
# Obtener el html de una clase / Get html from class
# ----------------------------------------------------------------------------------------------------
def get_htmlclass(classtype, classname):
    try:
        txtvalue = dom.find(classtype, class_=classname)
        return txtvalue.text
    except:
        return ''

# Inicializacion
# ----------------------------------------------------------------------------------------------------
original_stdout = sys.stdout # Save a reference to the original standard output
now = datetime.now()
step01 = '01-Links_'
step02 = '02-Comercios_'
step03 = '03-Menu'
filename = 'JustEat_' + now.strftime("%y-%m-%d_%H-%M-%S") + '_.ASC'
rootweb = 'https://www.just-eat.es'

# Estipular una navegador para realizar la solicitud
# ----------------------------------------------------------------------------------------------------
UAS = ("Mozilla/5.0 (Windows NT 6.1; WOW64; rv:40.0) Gecko/20100101 Firefox/40.1",
       "Mozilla/5.0 (Windows NT 6.3; rv:36.0) Gecko/20100101 Firefox/36.0",
       "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10; rv:33.0) Gecko/20100101 Firefox/33.0",
       "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2228.0 Safari/537.36",
       "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2227.1 Safari/537.36",
       "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2227.0 Safari/537.36",
       )

# headers = {'User-Agent': 'Mozilla/5.0'}

link = 'https://www.just-eat.es/restaurants-homeburgerbarmadrid/menu'
link = 'https://www.just-eat.es/restaurants-foodexpress/menu'

# ----------------------------------------------------------------------------------------------------
html_class = 'div'
name_class = 'tabs '

html_tbl_class = 'section'
name_tbl_class = 'menuCard-category accordion accordion--ruled is-open'

html_menu_class = 'div'
name_menu_class = 'menu-product product u-separated--dash'

# ----------------------------------------------------------------------------------------------------
with open('02-Comercios_JustEat_20-08-09_12-37-18_.ASC', 'r', encoding='utf8') as f:
    cnt = 1
    link = f.readline()

    while link:
        record = link.split("|")
        link = rootweb + record[2] + '/'
        Id_Comercio = record[1]

        # wait next read
        time.sleep(10)
#  Load the webpage content
# ----------------------------------------------------------------------------------------------------
        ua = UAS[random.randrange(len(UAS))]
        headers = {'user-agent': ua}
        r = requests.get(link, headers=headers, timeout=(3.05, 27))

# Comprobamos que la petición nos devuelve un Status Code = 200
# ----------------------------------------------------------------------------------------------------
        status_code = r.status_code
        if status_code == 200:

            # Convert to a beautiful soup object
            soup = bs(r.content, 'html.parser')

# links = soup.find_all('a', href=True)
# ----------------------------------------------------------------------------------------------------
            links = []
            div = soup.find(html_class, class_=name_class)
            if div:
                for a in div.find_all('a'):
                    links.append(a['href'])

            if not links:
                print('Error: No encontro class:' + name_class + ' Linea:'+str(cnt) + ' Comercio:' + Id_Comercio)

#            for link in links:
# Recolectar datos del Menu
# ----------------------------------------------------------------------------------------------------
            if link.find('menu') > 0 or 1==1:
                sections = soup.findAll(html_tbl_class, class_=name_tbl_class)

                if sections:
                    with open(step03 + filename, 'a+', encoding='utf8') as sf:
                        for section in sections:
                            dom = section
                            Col01 = get_htmlclass('h3', 'menuCard-category-title gamma accordion-header icon')

                            items = section.findAll(html_menu_class, class_=name_menu_class)
                            if items:
                                for item in items:
                                    dom = item
                                    Prod_Id = item.get('data-product-id')

                                    Col02 = get_htmlclass('h4', 'product-title')
                                    Col03 = get_htmlclass('div', 'product-description')
                                    Col04 = get_htmlclass('div', 'product-price u-noWrap')

                                    Prod_Att_02 = ''
                                    Prod_Att_03 = ''

                                    dom = item.find('button', 'btn btn--primary btn--roundedIcon btn-addproduct')
                                    if dom:
                                        Prod_Att_01 = 'true'
                                    else:
                                        Prod_Att_01 = 'false'
                                    dom = item.find('i', 'icon icon-product icon-product--spicy')
                                    if dom:
                                        Prod_Att_02 = dom.get('title')
                                    dom = item.find('i', 'icon icon-product icon-product--vegetarian')
                                    if dom:
                                        Prod_Att_03 = dom.get('title')

                                    TxtRow = Id_Comercio + '|' + Prod_Id + '|' + Prod_Att_01 + '|' +  Prod_Att_02 + '|' +  Prod_Att_03 + '|' + now.strftime("%Y-%m-%d") + '|' + Col01 + '|' +  Col02 + '|' +  Col03 + '|' +  Col04
                                    TxtRow = TxtRow.replace('\n', '-').replace('\r', ';') #+ ' \n'

                                    sf.write(TxtRow + ' \n')

                        sf.close()
                else:
                    print('Error: No existen seccions Linea:' + str(cnt) + ' Comercio:' + Id_Comercio)
            else:
                    print('Warning: Sin menu Linea:' + str(cnt) + ' Comercio:' + Id_Comercio)
        cnt += 1
        link = f.readline()

    else:
        print('Error: No encontro pagina HTML Linea:'+ str(cnt) + ' Comercio:' + Id_Comercio)

f.close()


from django.template import loader
from django.http import HttpResponse
from .models import Company
import json
import logging

logger = logging.getLogger('custom')

def index(request):
    template = loader.get_template('company/html/index.html')
    context = {
    }
    return HttpResponse(template.render(context, request))

def get_basic_info(request):
    company = Company.objects.all()[0]
    basic_info = {
        'company_name': company.company_name,
        'address': company.address,
        'telephone': company.telephone,
        'email': company.email,
        'website': company.website
    }
    return HttpResponse(json.dump(basic_info), content_type='application/json')

def get_introduction(request):
    company = Company.objects.all()[0]
    advanced_info = {
        'introduction': company.introduction
    }
    return HttpResponse(json.dump(advanced_info), content_type='application/json')

def get_business_scope(request):
    company = Company.objects.all()[0]
    advanced_info = {
        'business_scope': company.business_scope
    }
    return HttpResponse(json.dump(advanced_info), content_type='application/json')

def set_basic_info(request):
    if request.method == 'POST':
        try:
            company_name = request.POST['company_name']
            address = request.POST['address']
            telephone = request.POST['telephone']
            email = request.POST['email']
            website = request.POST['website']

            company = Company.objects.all()[0]
            company.company_name = company_name
            company.address = address
            company.telephone = telephone
            company.email = email
            company.website = website
            company.save()
            HttpResponse.status_code = 200
        except Exception as exception:
            logger.error(exception)
            HttpResponse.status_code = 500
            params = {
                "message": exception
            }
    else:
        HttpResponse.status_code = 405
        params = {
            "message": "request method should be POST"
        }
    return HttpResponse(json.dump(params), content_type='application/json')

def set_introduction(request):
    if request.method == 'POST':
        try:
            introduction = request.POST['introduction']
            company = Company.objects.all()[0]
            company.introduction = introduction
            company.save()
            HttpResponse.status_code = 200
        except Exception as exception:
            logger.error(exception)
            HttpResponse.status_code = 500
            params = {
                "message": exception
            }
    else:
        HttpResponse.status_code = 405
        params = {
            "message": "request method should be POST"
        }
    return HttpResponse(json.dump(params), content_type='application/json')

def set_business_scope(request):
    if request.method == 'POST':
        try:
            business_scope = request.POST['business_scope']
            company = Company.objects.all()[0]
            company.business_scope = business_scope
            company.save()
            HttpResponse.status_code = 200
        except Exception as exception:
            logger.error(exception)
            HttpResponse.status_code = 500
            params = {
                "message": exception
            }
    else:
        HttpResponse.status_code = 405
        params = {
            "message": "request method should be POST"
        }
    return HttpResponse(json.dump(params), content_type='application/json')


import asyncio
import utils
from .utils import UtilsTask
from reqs.live_daily_job import (
    HeartBeatReq,
    RecvHeartGiftReq,
    OpenSilverBoxReq,
    RecvDailyBagReq,
    SignReq,
    WatchTvReq,
    SignFansGroupsReq,
    SendGiftReq,
    ExchangeSilverCoinReq,
)

class HeartBeatTask:
    @staticmethod
    def target(step):
        if step == 0:
            return HeartBeatTask.heart_beat
        return None

    @staticmethod
    async def heart_beat(user):
        json_rsp = await user.req_s(HeartBeatReq.pc_heartbeat, user)
        # print(json_rsp)
        user.infos(['心跳包(5分钟左右间隔)'])
        json_rsp = await user.req_s(HeartBeatReq.app_heartbeat, user)
        print(json_rsp)
        return (0, (280, 300), user.id),

class RecvHeartGiftTask:
    @staticmethod
    def target(step):
        if step == 0:
            return RecvHeartGiftTask.recv_heartgift
        return None

    @staticmethod
    async def recv_heartgift(user):
        json_rsp = await user.req_s(RecvHeartGiftReq.recv_heartgift, user)
        if json_rsp['code'] == 400:
            user.fall_in_jail()
            return (0, (0, 0), user.id),
        return (0, (280, 300), user.id),

class OpenSilverBoxTask:
    @staticmethod
    def target(step):
        if step == 0:
            return OpenSilverBoxTask.open_silver_box
        return None

    @staticmethod
    async def open_silver_box(user):
        while True:
            user.infos(["检查宝箱状态"])
            temp = await user.req_s(OpenSilverBoxReq.check_time, user)
            # print (temp['code'])    #宝箱领完返回的code为-10017
            if temp['code'] == -10017:
                user.infos(["今日宝箱领取完毕"])
                json_rsp = None
            else:
                time_start = temp['data']['time_start']
                time_end = temp['data']['time_end']
                json_rsp = await user.req_s(OpenSilverBoxReq.open_silver_box, user, time_start, time_end)
            if json_rsp is None or json_rsp['code'] == -10017 or json_rsp['code'] == -800:
                sleeptime = utils.seconds_until_tomorrow() + 300
                return (0, (sleeptime, sleeptime+30), user.id),
            elif not json_rsp['code']:
                user.infos(["打开了宝箱"])
            elif json_rsp['code'] == 400:
                user.infos(["宝箱开启中返回了小黑屋提示"])
                user.fall_in_jail()
                # 马上继续调用，由下一次的user去supend这个task
                return (0, (0, 0), user.id),
            else:
                user.infos(["继续等待宝箱冷却..."])
                sleeptime = (json_rsp['data'].get('surplus', 3)) * 60 + 5
                return (0, (sleeptime, sleeptime+30), user.id),

class RecvDailyBagTask:
    @staticmethod
    def target(step):
        if step == 0:
            return RecvDailyBagTask.recv_dailybag
        return None

    @staticmethod
    async def recv_dailybag(user):
        json_rsp = await user.req_s(RecvDailyBagReq.recv_dailybag, user)
        try:
            json_rsp['data']['bag_list']
        except:
            user.warn(f'recv_dailybag {json_rsp}')
        for i in json_rsp['data']['bag_list']:
            user.infos([f'获得-{i["bag_name"]}-成功'])
        sleeptime = 21600
        return (0, (sleeptime, sleeptime+30), user.id),

class SignTask:
    @staticmethod
    def target(step):
        if step == 0:
            return SignTask.sign
        return None

    @staticmethod
    async def sign(user):
        json_rsp = await user.req_s(SignReq.sign, user)
        user.infos([f'签到状态: {json_rsp["msg"]}'])
        if json_rsp['code'] == -500 and '已' in json_rsp['msg']:
            sleeptime = utils.seconds_until_tomorrow() + 300
        else:
            sleeptime = 350
        return (0, (sleeptime, sleeptime+30), user.id),

class WatchTvTask:
    @staticmethod
    def target(step):
        if step == 0:
            return WatchTvTask.watch_tv
        return None

    @staticmethod
    async def watch_tv(user):
        # -400 done/not yet
        json_rsp = await user.req_s(WatchTvReq.watch_tv, user)
        user.infos([f'双端观看直播:  {json_rsp["msg"]}'])
        if json_rsp['code'] == -400 and '已' in json_rsp['msg']:
            sleeptime = utils.seconds_until_tomorrow() + 300
        else:
            sleeptime = 350
        return (0, (sleeptime, sleeptime+30), user.id),

class SignFansGroupsTask:
    @staticmethod
    def target(step):
        if step == 0:
            return SignFansGroupsTask.sign_groups
        return None

    @staticmethod
    async def sign_groups(user):
        json_rsp = await user.req_s(SignFansGroupsReq.fetch_groups, user)
        groups = [(i['group_id'], i['owner_uid']) for i in json_rsp['data']['list']]
        if groups:
            tasks = []
            for group_id, owner_uid in groups:
                task = asyncio.ensure_future(SignFansGroupsTask.sign_group(user, group_id, owner_uid))
                tasks.append(task)
            await asyncio.wait(tasks, return_when=asyncio.ALL_COMPLETED)
        sleeptime = 21600
        return (0, (sleeptime, sleeptime+30), user.id),

    @staticmethod
    async def sign_group(user, group_id, owner_uid):
        json_rsp = await user.req_s(SignFansGroupsReq.sign_group, user, group_id, owner_uid)
        if not json_rsp['code']:
            data = json_rsp['data']
            if data['status']:
                user.infos([f'应援团 {group_id} 已应援过'])
            else:
                user.infos([f'应援团 {group_id} 应援成功,获得 {data["add_num"]} 点亲密度'])
        else:
            user.infos([f'应援团 {group_id} 应援失败'])

class SendGiftTask:
    @staticmethod
    def target(step):
        if step == 0:
            return SendGiftTask.send_gift
        return None

    @staticmethod
    async def fetch_giftbags(user):
        gift_bags = await UtilsTask.fetch_giftbags(user)
        results = []
        for bag_id, gift_id, gift_num, _, _, left_time in gift_bags:
            results.append([gift_id, gift_num, bag_id, left_time])
        return results

    @staticmethod
    async def fetch_wearing_medal(user):
        json_rsp = await user.req_s(SendGiftReq.fetch_wearing_medal, user)
        if not json_rsp['code']:
            data = json_rsp['data']
            if data:
                room_id = data['roominfo']['room_id']
                remain_intimacy = int(data['day_limit']) - int(data['today_feed'])
                medal_name = data['medal_name']
                return [(room_id, remain_intimacy, medal_name), ]
            else:
                # print('暂无佩戴任何勋章')
                return []

    @staticmethod
    async def send_expiring_gift(user):
        if not user.task_ctrl['clean-expiring-gift']:
            return
        gift_bags = await SendGiftTask.fetch_giftbags(user)
        room_id = user.task_ctrl['clean-expiring-gift2room']
        time_set = user.task_ctrl['set-expiring-time']
        # filter 用过期来过滤
        expiring_giftbags = []
        for gift in gift_bags:
            left_time = gift[3]
            if left_time is not None and 0 < int(left_time) < time_set:
                expiring_giftbags.append(gift[:3])
        if expiring_giftbags:
            print('发现即将过期的礼物')
            if user.task_ctrl['clean_expiring_gift2all_medal']:
                print('正在清理过期礼物到用户勋章')
                medals = await UtilsTask.fetch_medals(user)
                expiring_giftbags = await SendGiftTask.fill_intimacy(user, expiring_giftbags, medals)

            print('正在清理过期礼物到指定房间')
            for gift_id, gift_num, bag_id in expiring_giftbags:
                await UtilsTask.send_gift(user, room_id, gift_num, bag_id, gift_id)
        else:
            print('未发现即将过期的礼物')

    @staticmethod
    async def send_medal_gift(user):
        medals = []
        if user.task_ctrl['send2wearing-medal']:
            medals = await SendGiftTask.fetch_wearing_medal(user)
            if not medals:
                print('暂未佩戴任何勋章')
        if user.task_ctrl['send2medal']:
            medals += await UtilsTask.fetch_medals(user, user.task_ctrl['send2medal'])
        # print('目前的勋章', medals)

        print('正在投递勋章')
        gift_bags = await SendGiftTask.fetch_giftbags(user)
        # print(temp)
        send_giftbags = []
        for gift in gift_bags:
            gift_id = int(gift[0])
            left_time = gift[3]
            # 过滤某些特定礼物以及永久礼物
            if (gift_id not in [4, 3, 9, 10]) and left_time is not None:
                send_giftbags.append(gift[:3])
        await SendGiftTask.fill_intimacy(user, send_giftbags, medals)

    @staticmethod
    async def fill_intimacy(user, gift_bags, medals):
        json_rsp = await user.req_s(SendGiftReq.fetch_gift_config, user)
        gift_price = {gift['id']: (gift['price'] / 100) for gift in json_rsp['data']}
        num_sent = 0
        for room_id, remain_intimacy, medal_name in medals:
            filled_intimacy = 0
            for gift in gift_bags:
                gift_id, gift_num, bag_id = gift
                if (gift_num * gift_price[gift_id] <= remain_intimacy):
                    num_sent = gift_num
                elif remain_intimacy >= gift_price[gift_id]:
                    num_sent = int(remain_intimacy / gift_price[gift_id])
                else:
                    continue
                gift[1] -= num_sent
                score = gift_price[gift_id] * num_sent
                await UtilsTask.send_gift(user, room_id, num_sent, bag_id, gift_id)
                filled_intimacy += score
                remain_intimacy -= score
            user.infos([f'对 {medal_name} 共送出亲密度为{filled_intimacy}的礼物'])
        # 过滤掉送光了的礼物包
        return [gift for gift in gift_bags if gift[1]]

    @staticmethod
    async def send_gift(user):
        await SendGiftTask.send_medal_gift(user)
        await SendGiftTask.send_expiring_gift(user)
        sleeptime = 21600
        return (0, (sleeptime, sleeptime+30), user.id),

class ExchangeSilverCoinTask:
    @staticmethod
    def target(step):
        if step == 0:
            return ExchangeSilverCoinTask.silver2coin
        return None

    @staticmethod
    async def silver2coin(user):
        if not user.task_ctrl['silver2coin']:
            sleeptime = 21600
            return (0, (sleeptime, sleeptime+30), user.id),
        json_rsp = await user.req_s(ExchangeSilverCoinReq.silver2coin_web, user)
        user.infos([f'{json_rsp["msg"]}'])
        if json_rsp['code'] == 403 and '最多' in json_rsp['msg']:
            finish_web = True
        else:
            finish_web = False
        if finish_web:
            sleeptime = utils.seconds_until_tomorrow() + 300
        else:
            sleeptime = 350

        return (0, (sleeptime, sleeptime+30), user.id),


import time
import os
import sys
sys.path.append(os.getcwd())
import re
#import logger
import requests
from requests.packages.urllib3.exceptions import InsecureRequestWarning
requests.packages.urllib3.disable_warnings(InsecureRequestWarning)
import yaml
import json
import numpy as np
import numpy.testing as npt
from loguru import logger
from util.redis_producer import redis_producer
#from multiprocessing import Lock
import threading
#logger = logger.getLogger()
#logger.setLevel(logger.INFO)

class TaskRunnerAPI:
    conf = None
    token = None
    def __init__(self):
        self.conf = {}
        logger.info("runner init")
        self.conf["site"] = os.getenv("consolesite")
        self.conf["user"] = os.getenv("consoleuser")
        self.conf["passwd"] = os.getenv("consolepasswd")
        #fr = open('./taskrunner/conf.yml')
        #self.conf = yaml.load(fr)
        #logger.info(self.conf)
        #fr.close()
        self.MQ = redis_producer()

    def produces(self,api,time=None,isOK=True):
        return
        tmp = {}
        tmp["api"] = api
        if isOK == False:
            tmp["isOK"] = False
        else:
            tmp["isOK"] = True
            tmp["time"] = time
        self.MQ.producer(json.dumps(tmp))

    def pub_msg(self,msg):
        return
        self.MQ.pub_performace(msg=msg)

    def run(self,code,name=None):
        logger.info(threading.current_thread().name)
        #return True
        if not self.login():
            logger.error("login failed")
            return False
        aliveds = self.find_allalive_ds()
        if aliveds == [] or aliveds == False:
            logger.error("no alive ds")
            return False
        datasource = self.find_metadata(aliveds)
        if not datasource:
            logger.error("no data")
            return False

        logger.info("running job")
        #[{""resultVarName"":""result"",""resultDest"":""-1""}]
        var_result = self.code_reveal(code=code)
        result = []
        for var in var_result:
            tmp = {
                "resultVarName":var,
                "resultDest" : datasource.get("dataServerId")
            }
            result.append(tmp)
        datasource_l = []
        datasource["varName"] = "data"
        datasource_l.append(datasource)
        return  self.job_pipline(datasource=[] ,result=result,code=self.sub_debug_reveal(code=code),name=name)

    def code_reveal(self,code):
        print(code.split('\n'))
        re_reveal = re.compile(r'''^[a-zA-Z_\-0-9\ \=]+\.(?:debug_reveal|reveal)\([a-zA-Z_0-9\ \[\]\-\.\(\),=]+['"]([a-zA-Z_\-0-9]+)['"]''')
        reveal = []
        for line in code.split('\n'):
            tmp = re.search(re_reveal,line)
            if tmp:
                print(line)
                reveal.append(tmp.group(1))
        return  reveal

    def sub_debug_reveal(self,code):
        re_reveal = re.compile(r'\.debug_reveal\(')
        return re.sub(re_reveal,".reveal(",code)

    def job_pipline(self,datasource,result,code,name=None):
        logger.info("start job")
        begin = time.time()
        jobid = self.job_create(datasource=datasource,result=result,code=code,name=name)
        if jobid == False:
            return False
        if not self.job_start(jobid=jobid):
            return False
        if not  self.job_status(jobid=jobid):
            return  False
        result =  self.job_result(jobid=jobid)
        end = time.time()
        if len(result.keys()) != 0:
            msg = {}
            msg["jobname"] = name
            msg["time"]    = end-begin
            self.pub_msg(msg=msg)
            #self.job_delete(jobid=jobid)
        return result

    def job_create(self,datasource,result,code,name=None):
        if name == None:
            jobname = 'privpy-libarary'
        else:
            jobname = 'privpy-libarary-'+name
        body = {
            "name":jobname,
            "code":code,
            "taskDataSourceVOList":datasource,
            "taskResultVOList":result
        }
        head = {
            "Authorization": "bearer %s" % (self.token),
            "Content-Type": "application/json"
        }
        logger.info(body)
        try:
            if "http://" in self.conf.get("site"):
                url = "%s/api/api-tm/task" % self.conf.get("site")
            else:
                url = "https://%s/api/api-tm/task" %(self.conf.get("site"))
            while 1:
                req = requests.post(url=url,headers=head,data=json.dumps(body),verify=False)
                self.produces(api="POST/api/api-tm/task", time=req.elapsed.total_seconds() * 1000, isOK=True)
                logger.info(req.text)
                if req.json().get("subCode") == 'GLOBAL0004':
                    logger.info(req.text)
                    time.sleep(1)
                    self.login()
                    head["Authorization"] = "bearer %s" % (self.token)
                    continue
                if req.json().get("code") != 0:
                    logger.error("create job failed :%s"%(req.text))
                    return False
                return req.json().get("data").get("id")
        except Exception as err:
            self.produces(api="POST/api/api-tm/task", isOK=False)
            logger.error(err)
            logger.error(req.text)
            return False
    def job_start(self,jobid):
        logger.info("start job:%s"%(str(jobid)))
        head = {
            "Authorization": "bearer %s" % (self.token),
            "Content-Type": "application/json"
        }

        try:
            if "http://" in self.conf.get("site"):
                url = "%s/api/api-tm/task/startTask" % self.conf.get("site")
            else:
                url = "https://%s/api/api-tm/task/startTask" %(self.conf.get("site"))
            url = "%s/%s"%(url,str(jobid))
            while 1:
                req = requests.put(url=url,headers=head,verify=False)
                self.produces(api="PUT/api/api-tm/task/startTask", time=req.elapsed.total_seconds() * 1000, isOK=True)
                logger.info(req.text)
                if req.json().get("subCode") == 'GLOBAL0004':
                    time.sleep(1)
                    logger.info(req.text)
                    self.login()
                    head["Authorization"] = "bearer %s" % (self.token)
                    continue
                if req.json().get("code") != 0:
                    logger.error("start job failed:%s"%(req.text))
                    return False
                else:
                    return True
        except Exception as err:
            self.produces(api="PUT/api/api-tm/task/startTask", isOK=False)
            logger.error(err)
        return True

    def job_delete(self,jobid):
        logger.info("delete job :%s" % (str(jobid)))
        head = {
            "Authorization": "bearer %s" % (self.token),
            "Content-Type": "application/json"
        }
        if "http://" in self.conf.get("site"):
            url = "%s/api/api-tm/task" % self.conf.get("site")
        else:
            url = "https://%s/api/api-tm/task" % (self.conf.get("site"))
        url = '%s/%s' % (url, str(jobid))
        try:
            while 1:
                req = requests.delete(url=url, headers=head, verify=False)
                self.produces(api="DELETE/api/api-tm/task", time=req.elapsed.total_seconds() * 1000, isOK=True)
                logger.info(req.text)
                if req.json().get("subCode") == 'GLOBAL0004':
                    time.sleep(1)
                    logger.info(req.text)
                    self.login()
                    head["Authorization"] = "bearer %s" % (self.token)
                    continue
                else:
                    return True

        except Exception as err:
            self.produces(api="DELETE/api/api-tm/task",  isOK=False)
            logger.error(err)
            logger.error(req.text)
            return False

    def job_status(self,jobid):
        logger.info("check job status:%s"%(str(jobid)))
        head = {
            "Authorization": "bearer %s" % (self.token),
            "Content-Type": "application/json"
        }
        if "http://" in self.conf.get("site"):
            url = "%s/api/api-tm/task" % self.conf.get("site")
        else:
            url = "https://%s/api/api-tm/task" % (self.conf.get("site"))
        url = '%s/%s' % (url, str(jobid))
        tryCount = 0
        try:
            while tryCount < 300:
                req = requests.get(url=url,headers=head,verify=False)
                self.produces(api="GET/api/api-tm/task", time=req.elapsed.total_seconds() * 1000, isOK=True)
                logger.info(req.text)
                if req.json().get("subCode") == 'GLOBAL0004':
                    time.sleep(1)
                    logger.info(req.text)
                    self.login()
                    head["Authorization"] = "bearer %s" % (self.token)
                    continue
                if req.json().get("data").get("queueStatus") == 6 or req.json().get("data").get("queueStatus") == 7  or req.json().get("data").get("queueStatus") == 8:
                    logger.info("job finished")
                    return True
                logger.info("job:%s is running ...."%(str(jobid)))
                time.sleep(6)
                tryCount =  tryCount + 1

        except Exception as err:
            self.produces(api="GET/api/api-tm/task", isOK=False)
            logger.error(err)
            logger.error(req.text)
            return False

    def job_result(self,jobid):
        logger.info("get job result")
        if "http://" in self.conf.get("site"):
            url = "%s/api/api-tm/task/getTaskResult" % self.conf.get("site")
        else:
            url = "https://%s/api/api-tm/task/getTaskResult" % (self.conf.get("site"))
        url = '%s/%s' % (url, str(jobid))
        head = {
            "Authorization": "bearer %s" % (self.token),
            "Content-Type": "application/json"
        }
        result_ditc = {}
        try:
            get_result = True
            while 1:
                get_result = True
                req = requests.get(url=url,headers=head,verify=False)
                self.produces(api="GET/api/api-tm/task/getTaskResult", time=req.elapsed.total_seconds() * 1000, isOK=True)
                logger.info(req.text)
                if req.json().get("subCode") == 'GLOBAL0004':
                    time.sleep(1)
                    logger.info(req.text)
                    self.login()
                    head["Authorization"] = "bearer %s" % (self.token)
                    continue
                for res in req.json().get("data"):
                    tmp = {}
                    #logger.info(type(eval(res.get("result"))))
                    if res.get("result") == None:
                        get_result = False
                    try:
                        if isinstance((eval(res.get("result"))),list):
                            logger.info("list result")
                            tmp['val'] = np.array(eval(res.get("result")),dtype=float)
                            logger.info(tmp['val'].dtype)
                        else:
                            tmp['val'] = eval(res.get("result"))
                        result_ditc[res.get("resultVarName")] = tmp
                    except Exception as err:
                        logger.info(err)
                        tmp['val'] = res.get("result")
                        result_ditc[res.get("resultVarName")] = tmp
                if get_result == False:
                    time.sleep(3)
                    continue
                return result_ditc
        except Exception as err:
            self.produces(api="GET/api/api-tm/task/getTaskResult",  isOK=False)
            logger.error(err)
            logger.error(req.text)
            return result_ditc

    def login(self):
        threadname = threading.current_thread().name
        logger.info("login :%s"%(self.conf.get("site")))
        if "http://" in self.conf.get("site"):
            url = "%s/api/api-sso/token/simpleLogin" % self.conf.get("site")
        else:
            url = "https://%s/api/api-sso/token/simpleLogin" % (self.conf.get("site"))
        if 'multiRun' in threadname:
            username = self.conf.get("user")+str((int(threadname.split('_')[-1])+1)%(int(self.conf.get("thread"))))
        else:
            username = self.conf.get("user")
        data = "username=%s&password=%s" % (username, self.conf.get("passwd"))
        logger.info(data)
        try:
            req = requests.post(url=url, params=data, verify=False)
            # print(req.text)
            self.token = req.json().get("data").get("access_token")
            self.produces(api="POST/api/api-sso/token/simpleLogin", time=req.elapsed.total_seconds() * 1000, isOK=True)
        except Exception as err:
            logger.error(err)
            self.produces(api="POST/api/api-sso/token/simpleLogin",  isOK=False)
            return False
        return  True

    def find_allalive_ds(self):
        logger.info("find ds")
        if "http://" in self.conf.get("site"):
            url = "%s/api/api-tm/dataServer" % self.conf.get("site")
        else:
            url = "https://%s/api/api-tm/dataServer" % (self.conf.get("site"))
        head = {
            "Authorization": "bearer %s" % (self.token),
            "Content-Type": "application/json"
        }
        alive_ds = []
        page = 1
        try:
            while 1:
                req = requests.get(url='%s?page=%s'%(url,str(page)),headers=head,verify=False)
                self.produces(api="GET/api/api-tm/dataServer", time=req.elapsed.total_seconds() * 1000, isOK=True)
                logger.info(req.text)
                if req.json().get("subCode") == 'GLOBAL0004':
                    time.sleep(1)
                    logger.info(req.text)
                    self.login()
                    head["Authorization"] = "bearer %s" % (self.token)
                    continue
                if len(req.json().get("data").get("data")) == 0:
                    logger.error("ds is empty")
                    return False
                for ds in req.json().get("data").get("data"):
                    if ds.get("status") == '1' and ds.get("isCache") == 0:
                        alive_ds.append(ds.get("id"))
                if page == req.json().get("data").get("nextPageNo"):
                    return  alive_ds
                page = page + 1
        except Exception as err:
            self.produces(api="GET/api/api-tm/dataServer", isOK=False)
            logger.error(req.text)
            logger.error(err)
            return False

    def find_metadata(self,aliveds = []):
        datasource = {}
        logger.info("find metadata")
        if "http://" in self.conf.get("site"):
            url = "%s/api/api-tm/dataSourceMetadata" % self.conf.get("site")
        else:
            url = "https://%s/api/api-tm/dataSourceMetadata" % (self.conf.get("site"))
        head = {
            "Authorization": "bearer %s" % (self.token),
            "Content-Type": "application/json"
        }
        page = 1
        try:
            while 1:
                req = requests.get(url='%s?page=%s'%(url,str(page)),headers=head,verify=False)
                self.produces(api="GET/api/api-tm/dataSourceMetadata", time=req.elapsed.total_seconds() * 1000, isOK=True)
                logger.info(req.text)
                if req.json().get("subCode") == 'GLOBAL0004':
                    time.sleep(1)
                    logger.info(req.text)
                    self.login()
                    head["Authorization"] = "bearer %s" % (self.token)
                    continue
                if len(req.json().get("data").get("data")) == 0:
                    logger.info("data is empty")
                    return False
                for data in req.json().get("data").get("data"):
                    if data.get("dsId")  in aliveds:
                        datasource["dataSourceId"] = req.json().get("data").get("data")[0].get("dataSetId")
                        datasource["dataSourceMetadataId"] = req.json().get("data").get("data")[0].get("id")
                        datasource["dataServerId"] = req.json().get("data").get("data")[0].get("dsId")
                        return  datasource
                if page == req.json().get("data").get("nextPageNo"):
                    return False
                page = page + 1
        except Exception as err:
            self.produces(api="GET/api/api-tm/dataSourceMetadata", isOK=True)
            logger.error(err)
            logger.error(req.text)
            return False

if __name__ == '__main__':
    runner = TaskRunnerAPI()
    code= '''
import privpy as pp
import os
import sys
sys.path.append(os.getcwd() + '/privpy_lib')
import pnumpy as pnp
import numpy as np

a = pp.farr([1, 2])
b = pp.farr([1, 2, 3])
c = pp.farr([3, 4])
d = pp.farr([1, 3])

s1 = pnp.array_equal(a, a)
s2 = pnp.array_equal(a, b)
s3 = pnp.array_equal(a, c)
s4 = pnp.array_equal(a, d)

pp.debug_reveal(s1, 's1')
pp.debug_reveal(s2, 's2')
pp.debug_reveal(s3, 's3')
pp.debug_reveal(s4, 's4')
    '''
    #print(runner.sub_debug_reveal(code))
    #print(runner.code_reveal(code=code))
    res = (runner.run(code=code))
    print(res)
    a = np.array([1, 2])
    b = np.array([1, 2, 3])
    c = np.array([3, 4])
    d = np.array([1, 3])

    #res = test_util.run_task(code)
    s1 = res['s1']['val']
    s2 = res['s2']['val']
    s3 = res['s3']['val']
    s4 = res['s4']['val']

    npt.assert_array_almost_equal(s1, np.array_equal(a, a))
    npt.assert_array_almost_equal(s2, np.array_equal(a, b))
    npt.assert_array_almost_equal(s3, np.array_equal(a, c))
    npt.assert_array_almost_equal(s4, np.array_equal(a, d))
    print('assert ok')


import sys,os
import numpy as np
import time as stopWatch
from netCDF4 import Dataset
from datetime import date, timedelta
from collections import defaultdict

#TAKE INPUTS FROM SHELL SCRIPT
#Process inputs and call master function
inputData = sys.argv[1:] #exclude 1st item (script name)
year = int(inputData[0])
powSys = str(inputData[1]) #wecc or ercot
print('Running year:' + str(year) + ' and system: ' + powSys,flush=True)

#DEFINE PARAMETERS
#start & end dates for dataset
start_date = date(year, 1, 1)
end_date = date(year, 12, 31)
#Setup for loading the data through each day by day dataset FILE NAME for the two different MERRA files
merraRoot = str("/scratch/mtcraig_root/mtcraig1/shared_data/merraData/resource/" + powSys)
rawDir = os.path.join(merraRoot,'raw')
if year <= 2010:
    baseWord1 = os.path.join(rawDir,"MERRA2_300.tavg1_2d_slv_Nx.")#for slv data
    baseWord2 = os.path.join(rawDir,"MERRA2_300.tavg1_2d_rad_Nx.")#for rad data
if year > 2010:
    baseWord1 = os.path.join(rawDir,"MERRA2_400.tavg1_2d_slv_Nx.")
    baseWord2 = os.path.join(rawDir,"MERRA2_400.tavg1_2d_rad_Nx.")
#Directory in which to store files
processDir = os.path.join(merraRoot,'processed')
fileName = os.path.join(processDir,"processedMERRA" + powSys + str(year) + ".nc")

#method to save each coordinate in its own file with its own variables
def rewriteData(data1, variableTag):
    data = np.zeros((len(data1.variables[variableTag]),(len(data1.variables[variableTag][0])*len(data1.variables[variableTag][0][0]))))
    timeValueIndex = -1
    latLongIndex = 0
    for time in data1.variables[variableTag]:
        timeValueIndex += 1
        latLongIndex = 0
        for lat in time:
            for long in lat:
                data[timeValueIndex][latLongIndex] = long
                latLongIndex += 1
    return data

#timestep leave at 1 day
delta = timedelta(days=1)

#Creating datsets for all the different variables
masterDataset = defaultdict(list)

daysPassed = 0
previousYear = start_date.year

#stop watch to see how long certain program parts take
start_timeExtract = stopWatch.time()
start_timeToal = stopWatch.time()

# ***************** IJBD EDIT (PASS LAT/LONS) ***************** #
date = start_date.strftime("%Y%m%d")
filename = baseWord1 + date + ".SUB.nc" #.nc4.nc4
data = Dataset(filename)
lats = np.array(data.variables['lat'][:])
lons = np.array(data.variables['lon'][:])
data.close()
# ************************************************************* #

#main loop to iterate through the different day netcdf files
while start_date <= end_date:

    #checking for leap day then incremanting to the next day (skipping it) if it is a leap day
    if(start_date.year % 4 == 0 and start_date.month == 2 and start_date.day == 29):
        start_date += delta

    date = start_date.strftime("%Y%m%d")
    filename1 = baseWord1 + date + ".SUB.nc" #.nc4.nc4
    filename2 = baseWord2 + date + ".SUB.nc" # .nc4.nc4

    data1 = Dataset(filename1)
    data2 = Dataset(filename2)

    #adding new data from netcdf day files to specific lat long cords

    #Eastward winds at 2,10, and 50 meters
    masterDataset["U2M"].append(rewriteData(data1,"U2M"))
    masterDataset["U10M"].append(rewriteData(data1,"U10M"))
    masterDataset["U50M"].append(rewriteData(data1,"U50M"))

    #Northward winds at 2,10, and 50 meters
    masterDataset["V2M"].append(rewriteData(data1,"V2M"))
    masterDataset["V10M"].append(rewriteData(data1,"V10M"))
    masterDataset["V50M"].append(rewriteData(data1,"V50M"))

    #Surface Temperature at 2 meters
    masterDataset["T2M"].append(rewriteData(data1,"T2M"))

    #Surface Pressure
    masterDataset["PS"].append(rewriteData(data1,"PS"))

    #Specific humidity
    masterDataset["QV2M"].append(rewriteData(data1,"QV2M"))

    #surface_incoming_shortwave_flux
    masterDataset["SWGDN"].append(rewriteData(data2,"SWGDN"))

    data1.close()
    data2.close()

    #Removing day files after unloading the data into specific cords, optional may speed up runtime
    #os.remove(filename)
    #os.remove(filename2)
    daysPassed += 1
    start_date += delta
timeLevel = 0

#calculating extracting data length time
extractTimeLength = (stopWatch.time() - start_timeExtract)
start_timeRewrite = stopWatch.time()

print("Finished extracting data sets from " + str(daysPassed) + " days!")
print("It took " + str(round(extractTimeLength,2))  + " seconds to extract the data.")
print("Now writing lat and long specific files!")

#writing new netcdf format into file, change name and location
newNetCDF = Dataset(fileName, 'w')
print("netCDF file opened for writing the processed data")

#declaring dimensions: goes lat, long,  year-1980-2019, day 0-365, time 0-24
newNetCDF.createDimension('yearDayIndex',len(masterDataset["U2M"]))
newNetCDF.createDimension('time', len(masterDataset["U2M"][0]))
newNetCDF.createDimension('latLong',len(masterDataset["U2M"][0][0]))

print("Dimensions are declared as yearDayIndex, time, and product of latlong size")
# ************** IJBD EDITS (PASS LAT/LONS) *************** #

print("Writing Lat and Lon variables in the new netCDF4 file, and assigning data")
newNetCDF.createDimension('lat',len(lats))
newNetCDF.createDimension('lon',len(lons))
varlat = newNetCDF.createVariable("lat",'double', ("lat",))
varlon = newNetCDF.createVariable("lon",'double', ("lon",))
varlat[:] = lats
varlon[:] = lons

print("Lat and Lon variables in the new netCDF4 file have been created")
# ********************************************************* #

#Setting up the 10 different variables, goes latLong 0-4700, yearDay 0-daysPassed, time 0-24

print("Setting up the datatype for 10 variables - U2M, U10M< U50M< V2M, T2M, QV2M, PS, and SWGDN")

varU2M = newNetCDF.createVariable("U2M",'double', ('latLong','yearDayIndex','time'))
varU10M = newNetCDF.createVariable("U10M",'double', ('latLong','yearDayIndex','time'))
varU50M = newNetCDF.createVariable("U50M",'double', ('latLong','yearDayIndex','time'))
varV2M = newNetCDF.createVariable("V2M",'double', ('latLong','yearDayIndex','time'))
varV10M = newNetCDF.createVariable("V10M",'double', ('latLong','yearDayIndex','time'))
varV50M = newNetCDF.createVariable("V50M",'double', ('latLong','yearDayIndex','time'))
varT2M = newNetCDF.createVariable("T2M",'double', ('latLong','yearDayIndex','time'))
varQV2M = newNetCDF.createVariable("QV2M",'double', ('latLong','yearDayIndex','time'))
varPS= newNetCDF.createVariable("PS",'double', ('latLong','yearDayIndex','time'))
varSWGDN = newNetCDF.createVariable("SWGDN",'double', ('latLong','yearDayIndex','time'))

print("Data types have been assigned")

#loop for writing out cord file

print("starting while loop")
latLongIndex = 0
while latLongIndex < len(masterDataset["U2M"][0][0]):
    yearDayIndex = 0
    date = 0
    while yearDayIndex < len(masterDataset["U2M"]): #get each day from the master list
            timeLevel = 0
            while timeLevel < len(masterDataset["U2M"][0]):
                varU2M[latLongIndex,yearDayIndex,timeLevel] = masterDataset["U2M"][yearDayIndex][timeLevel][latLongIndex]
                varU10M[latLongIndex,yearDayIndex,timeLevel] = masterDataset["U10M"][yearDayIndex][timeLevel][latLongIndex]
                varU50M[latLongIndex,yearDayIndex,timeLevel] = masterDataset["U50M"][yearDayIndex][timeLevel][latLongIndex]
                varV2M[latLongIndex,yearDayIndex,timeLevel] = masterDataset["V2M"][yearDayIndex][timeLevel][latLongIndex]
                varV10M[latLongIndex,yearDayIndex,timeLevel] = masterDataset["V10M"][yearDayIndex][timeLevel][latLongIndex]
                varV50M[latLongIndex,yearDayIndex,timeLevel] = masterDataset["V50M"][yearDayIndex][timeLevel][latLongIndex]
                varT2M[latLongIndex,yearDayIndex,timeLevel] = masterDataset["T2M"][yearDayIndex][timeLevel][latLongIndex]
                varQV2M[latLongIndex,yearDayIndex,timeLevel] = masterDataset["QV2M"][yearDayIndex][timeLevel][latLongIndex]
                varPS[latLongIndex,yearDayIndex,timeLevel] = masterDataset["PS"][yearDayIndex][timeLevel][latLongIndex]
                varSWGDN[latLongIndex,yearDayIndex,timeLevel] = masterDataset["SWGDN"][yearDayIndex][timeLevel][latLongIndex]
                timeLevel+= 1
            yearDayIndex+= 1
    latLongIndex += 1

#Runtime for program
rewriteTimeLength = (stopWatch.time() - start_timeRewrite)
totalTimeLength = (stopWatch.time() - start_timeToal)
print("It took " + str(round(rewriteTimeLength,2))  + " seconds to rewrite older datasets!")#time to rewrite datsets and delete old files
print("It took " + str(round(totalTimeLength,2))  + " seconds to execute the whole program!")#total time for program to run

print("Finished while loop; data has beeen processed")


__description__ = \
"""
Wrapped interface to networkx allowing gpmap to access primitives for network
construction.
"""
__author__ = "Michael J. Harms"
__date__ = "2021-08-31"

__nx_dev_version__ = "2.6.2"

import networkx as nx
import matplotlib.pyplot as plt

import inspect, warnings

_NX_WARNING = \
"""
networkx function '{}' did not have the expected argument '{}'. This may
indicate an incompatibility between your versions of gpmap and networkx
versions.  gpamp was developed using networkx {}. You are using version {}.
Graphs may not turn out as expected.
"""

def _nx_wrapper(nx_function,ax,**kwargs):
    """
    This is a wrapper for NetworkX functions. NetworkX graphing functions have
    many parameters that change relatively often. This wrapper is an attempt at
    future-proofing. It uses `inspect` to figure out what parameters the
    function expects and then only passes parameters that will be recognized.
    If a parameter is not recognized, the function throws a warning which
    will hopefully alert the user to changes in the NetworkX API without
    causing our package to crash.  This function throws a ValueError if a
    required arugment is not passed.

    Parameters
    ----------
    nx_function : function
        a NetworkX function to wrap.
    ax : matplotlib.axes object
        axis on whichi to do plotting. this low-level function assumes you've
        already made the ax in some other context.
    **kwargs :
        kwargs arguments to pass to that function.

    returns ax, nx_fcn_return
    """

    required_args = {}
    args_with_defaults = {}

    # Figure out what networkx function expects
    fcn_sig = inspect.signature(nx_function)
    for p in fcn_sig.parameters:
        if fcn_sig.parameters[p].default is fcn_sig.parameters[p].empty:
            required_args[p] = None
        else:
            args_with_defaults[p] = fcn_sig.parameters[p]

    # Go through the kwargs passed in and construct input_kwargs.
    input_kwargs = {}
    input_kwargs["ax"] = ax

    bad_keys = []
    for k in kwargs:

        # First see if this is a required argument without a default.
        try:
            required_args.pop(k)
            input_kwargs[k] = kwargs[k]
            continue

        except KeyError:
            pass

        # Next see if this is an argument that has a default.
        try:
            args_with_defaults.pop(k)
            input_kwargs[k] = kwargs[k]
            continue

        except KeyError:
            pass

        bad_keys.append(k)

    # Make sure all required arguments were passed
    if len(required_args) > 0:
        err = f"Not all required arguments passed to '{nx_function}'\n"
        err += f"\nYour networkx version is {nx.__version__}\n"
        err += f"Missing required arguments are:\n"
        for a in required_args:
            err += f"    {a}\n"
        err += "\n"
        raise ValueError(err)

    # Warn if arguments were passed that were not recognized.
    if len(bad_keys) > 0:
        w = f"Not all keyword arguments were recognized by '{nx_function}'\n"
        w += f"\nYour networkx version is {nx.__version__}\n"
        w += f"Unrecognized arguments are:\n"
        for a in bad_keys:
            w += f"    {a}\n"
        w += "\n"

        if len(args_with_defaults) > 0:
            w += "\nNot all possible arguments were supplied. Did you mean to \n"
            w += "pass one of these (default given in parentheses)?\n"
            for a in args_with_defaults:
                w += f"    {a} ({args_with_defaults[a].default})\n"
            w += "\n\n"

        warnings.warn(w)

    return ax, nx_function(**input_kwargs)

def draw_nodes_docs():
    """
    Print full docs on draw_nodes().
    """

    print(f"\nYour networkx version is {nx.__version__}\n")
    print(inspect.cleandoc(nx.draw_networkx_nodes.__doc__))

def draw_edges_docs():
    """
    Print full docs on draw_edges().
    """

    print(f"\nYour networkx version is {nx.__version__}\n")
    print(inspect.cleandoc(nx.draw_networkx_edges.__doc__))

def draw_node_labels_docs():
    """
    Print full docs on draw_node_labels().
    """

    print(f"\nYour networkx version is {nx.__version__}\n")
    print(inspect.cleandoc(nx.draw_networkx_labels.__doc__))

def draw_edge_labels_docs():
    """
    Print full docs on draw_edge_labels().
    """

    print(f"\nYour networkx version is {nx.__version__}\n")
    print(inspect.cleandoc(nx.draw_networkx_edge_labels.__doc__))

def get_nx_kwargs(function,to_remove=None):
    """
    Get keyword arguments expected by a network function.

    Parameters
    ----------
    function : str
        function to pull from. should be one of "nodes", "edges", "node_labels",
        or "edge_labels"
    to_remove : list-like
        list of keyword arguments to remove. (These are typically arguments
        expected by some higher-level function. These can be removed when the
        kwargs are being extracted from networkx so the higher-level functions
        do not have to worry about networkx compatibility issues.)

    Returns
    -------
    kwargs : dict
        dictionary of keyword arguments with their default networkx values.
    """

    allowed_functions = {"nodes":nx.draw_networkx_nodes,
                         "edges":nx.draw_networkx_edges,
                         "node_labels":nx.draw_networkx_labels,
                         "edge_labels":nx.draw_networkx_edge_labels}

    try:
        nx_function = allowed_functions[function]
    except (KeyError,TypeError):
        err = "function '{function}' not recognized by get get_nx_kwargs.\n"
        err += "Should be one of:\n"
        for k in allowed_functions:
            err += f"    {k}\n"
        err += "\n"
        raise ValueError(err)

    # See if we're removing anything
    if to_remove is None:
        to_remove = []
    else:
        try:
            len(to_remove)
        except TypeError:
            err = "to_remove should be a list of keyword arguments to remove\n"
            err += "from the list of accessible kwargs\n"
            raise TypeError(err)

    # Get function signature
    kwargs = {}
    fcn_sig = inspect.signature(nx_function)
    for p in fcn_sig.parameters:
        if not fcn_sig.parameters[p].default is fcn_sig.parameters[p].empty:
            kwargs[p] = fcn_sig.parameters[p].default

    # Remove requested arguments
    for k in to_remove:
        try:
            kwargs.pop(k)
        except KeyError:
            w = _NX_WARNING.format(function,
                                   k,__nx_dev_version__,nx.__version__)
            warnings.warn(w)

    return kwargs

def draw_edges(G,pos,ax,**kwargs):
    """
    Draw edges on the graph.

    Parameters
    ----------
    G : GenotypePhenotypeGraph
        genotype phenotype graph object
    pos : dict
        A dictionary with nodes as keys and positions as values.
        Positions should be sequences of length 2.
    ax : matplotlib.axes
        axis on which to place plot
    kwargs :
        keyword arguments passed directly to draw_networkx_edges
        To see the args available run the convenience function
        draw_edges_docs().

    Note
    ----
    This uses the NetworkX `draw_networkx_edges` function. Rather that
    replicating their complex and not-the-same-forever API, all graphical
    arguments are passed directly via kwargs. To see the list of possible
    graphical inputs, see the networkx documentation.

    https://networkx.org/documentation/stable/reference/generated/networkx.drawing.nx_pylab.draw_networkx_edges.html
    """

    kwargs["G"] = G
    kwargs["pos"] = pos
    kwargs["ax"] = ax

    return _nx_wrapper(nx.draw_networkx_edges,**kwargs)

def draw_nodes(G,pos,ax,**kwargs):
    """
    Draw nodes on the graph.

    Parameters
    ----------
    G : GenotypePhenotypeGraph
        genotype phenotype graph object
    pos : dict
        A dictionary with nodes as keys and positions as values.
        Positions should be sequences of length 2.
    ax : matplotlib.axes
        axis on which to place plot
    kwargs :
        keyword arguments passed directly to draw_networkx_edges
        To see the args available run the convenience function
        draw_nodes_nodes().

    Note
    ----
    This uses the NetworkX `draw_networkx_nodes` function. Rather that
    replicating their complex and not-the-same-forever API, all graphical
    arguments are passed directly via kwargs. To see the list of possible
    graphical inputs, see the networkx documentation.

    https://networkx.org/documentation/stable/reference/generated/networkx.drawing.nx_pylab.draw_networkx_nodes.html
    """

    kwargs["G"] = G
    kwargs["pos"] = pos
    kwargs["ax"] = ax

    return _nx_wrapper(nx.draw_networkx_nodes,**kwargs)

def draw_node_labels(G,pos,ax,**kwargs):
    """
    Draw nodes on the graph.

    Parameters
    ----------
    G : GenotypePhenotypeGraph
        genotype phenotype graph object
    pos : dict
        A dictionary with nodes as keys and positions as values.
        Positions should be sequences of length 2.
    ax : matplotlib.axes
        axis on which to place plot
    kwargs :
        keyword arguments passed directly to draw_networkx_labels
        To see the args available run the convenience function
        draw_nodes_nodes().

    Note
    ----
    This uses the NetworkX `draw_networkx_labels` function. Rather that
    replicating their complex and not-the-same-forever API, all graphical
    arguments are passed directly via kwargs. To see the list of possible
    graphical inputs, see the networkx documentation.

    https://networkx.org/documentation/stable/reference/generated/networkx.drawing.nx_pylab.draw_networkx_labels.html
    """

    kwargs["G"] = G
    kwargs["pos"] = pos
    kwargs["ax"] = ax

    return _nx_wrapper(nx.draw_networkx_labels,**kwargs)

def draw_edge_labels(G,pos,ax,**kwargs):
    """
    Draw nodes on the graph.

    Parameters
    ----------
    G : GenotypePhenotypeGraph
        genotype phenotype graph object
    pos : dict
        A dictionary with nodes as keys and positions as values.
        Positions should be sequences of length 2.
    ax : matplotlib.axes
        axis on which to place plot
    kwargs :
        keyword arguments passed directly to draw_networkx_edges
        To see the args available run the convenience function
        draw_nodes_nodes().

    Note
    ----
    This uses the NetworkX `draw_networkx_edge_labels` function. Rather that
    replicating their complex and not-the-same-forever API, all graphical
    arguments are passed directly via kwargs. To see the list of possible
    graphical inputs, see the networkx documentation.

    https://networkx.org/documentation/stable/reference/generated/networkx.drawing.nx_pylab.draw_networkx_edge_labels.html
    """

    kwargs["G"] = G
    kwargs["pos"] = pos
    kwargs["ax"] = ax

    return _nx_wrapper(nx.draw_networkx_edge_labels,**kwargs)


import msgpack
import select
import socket
import struct

INF = 0x3f3f3f3f
DEFAULT_BUFFER_SIZE = 4096
DEFAULT_SOCKET_TIMEOUT = 5

def packmsg(msg):
    msg_bytes = msgpack.packb(msg, use_bin_type=True)
    meta_bytes = struct.pack('>I', len(msg_bytes))
    return meta_bytes + msg_bytes

def recvmsg(sock, timeout=DEFAULT_SOCKET_TIMEOUT, buf_size=DEFAULT_BUFFER_SIZE):
    rlist, wlist, xlist = select.select([sock], [], [], timeout)
    if len(rlist) == 0:
        raise socket.timeout('timeout when recv')
    buf_list = []
    total_len = INF
    read_len = 0
    sock.settimeout(DEFAULT_SOCKET_TIMEOUT)
    head = sock.recv(4)
    print(head)
    total_len = struct.unpack('>I', head)[0]
    while read_len < total_len:
        remain = total_len - read_len
        buf = sock.recv(min(remain, buf_size))
        if len(buf) == 0:
            break
        buf_list.append(buf)
        read_len += len(buf)
    return msgpack.unpackb(b''.join(buf_list), raw=False)

class TCPConnection:
    def __init__(self, host: str, port: int, timeout=DEFAULT_SOCKET_TIMEOUT):
        # NOTE just throw the error
        self.sock = socket.create_connection((host, port), timeout)

    def send(self, data: dict):
        self.sock.sendall(packmsg(data))

    def recv(self, timeout=DEFAULT_SOCKET_TIMEOUT, buf_size=DEFAULT_BUFFER_SIZE):
        return recvmsg(self.sock, timeout=timeout, buf_size=buf_size)

    def close(self):
        self.sock.close()

# REVIEW we don't have ca certificate for private ip, so we have to implement tcp encrypton ourselves.
# This can be done by mimic the TLS procedure
class SafeConnection:
    def __init__(self, hostname, port):
        pass