# -*- coding: utf-8 -*-
"""
Created on Sun Jan 26 23:01:56 2020

@author: Jake
"""

import torch
import numpy as np
from sklearn.model_selection import train_test_split
import matplotlib.pyplot as plt
from torch.autograd import grad
import time
from sklearn.preprocessing import StandardScaler
from imblearn.over_sampling import SMOTE

class Model(torch.nn.Module):
    def __init__(self):
        super(Model, self).__init__()
        self.linear_1 = torch.nn.Linear(20, 100)
        self.linear_2 = torch.nn.Linear(100,100)
        self.linear_3 = torch.nn.Linear(100,2)
        self.selu = torch.nn.SELU()
        self.softmax = torch.nn.Softmax(dim=1)

    def forward(self, x):
        x = self.linear_1(x)
        x = self.selu(x)
        x = self.linear_2(x)
        x = self.selu(x)
        x = self.linear_3(x)
        pred = self.softmax(x)

        return pred

def hessian_vector_product(ys,xs,v):
    J = grad(ys,xs, create_graph=True)[0]
    J.backward(v,retain_graph=True)
    return xs.grad

r = np.random.RandomState(seed=1234567890)

x_imputed = np.load('data/x_imputed.npy')
y = np.load('data/y.npy')

device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
if(torch.cuda.is_available()):
    print('Using device:', torch.cuda.get_device_name(torch.cuda.current_device()))

model = Model()
criterion = torch.nn.CrossEntropyLoss()
optimizer = torch.optim.SGD(model.parameters(), lr=0.05, momentum=0.9)

mlp_probs = np.array([])
test_labels = np.array([])
test_data = np.array([])

no_epochs = 2000
train_loss = list()
val_loss = list()
best_val_loss = 1

sm = SMOTE()

x_train,x_test,y_train,y_test = train_test_split(x_imputed,y,test_size=0.1,random_state=r)

scaler = StandardScaler()
x_train = scaler.fit_transform(x_train)
x_test = scaler.transform(x_test)

# case_idx = np.where(y_test==1)[0]
# x_test = x_test[case_idx]
# y_test = y_test[case_idx]

x_train, y_train = sm.fit_resample(x_train, y_train)

x_train = torch.from_numpy(x_train).float().to(device)
x_test = torch.from_numpy(x_test).float().to(device)
y_train = torch.from_numpy(y_train).long().to(device)
y_test = torch.from_numpy(y_test).long().to(device)

if device:
    model.to(device)
    print('Moved to GPU')

print_iter = 500

for epoch in range(no_epochs):
    total_train_loss = 0
    total_val_loss = 0

    model.train()
    # training

    image = x_train
    label = y_train

    optimizer.zero_grad()

    pred = model(image)

    loss = criterion(pred, label)
    total_train_loss += loss.item()

    loss.backward()
    optimizer.step()

    train_loss.append(total_train_loss)

    # validation
    model.eval()
    total = 0

    image = x_test
    label = y_test

    pred_test = model(image)

    loss = criterion(pred_test, label)
    total_val_loss += loss.item()

    val_loss.append(total_val_loss)
    if(epoch % print_iter == 0 or epoch == no_epochs-1):
        print('\nEpoch: {}/{}, Train Loss: {:.8f}, Test Loss: {:.8f}'.format(epoch + 1, no_epochs, total_train_loss, total_val_loss))

start_time = time.time()
model.zero_grad()

train_loss = criterion(model(x_train),y_train)

test_loss = criterion(model(x_test),y_test)

test_loss.backward(create_graph=True)

scale = 1000
damping = 1
num_samples = 1
recursion_depth=1000
print_iter = recursion_depth/10
v = model.linear_3.weight.grad.clone()
cur_estimate = v.clone()
for i in range(recursion_depth):
    hvp = hessian_vector_product(train_loss, model.linear_3.weight, cur_estimate)
    cur_estimate = [a+(1-damping)*b-c/scale for (a,b,c) in zip(v,cur_estimate,hvp)]
    cur_estimate = torch.squeeze(torch.stack(cur_estimate))#.view(1,-1)
    model.zero_grad()
    if (i % print_iter ==0) or (i==recursion_depth-1):
        numpy_est = cur_estimate.detach().cpu().numpy()
        numpy_est = numpy_est.reshape(1,-1)
        print("Recursion at depth %s: norm is %.8lf" % (i,np.linalg.norm(np.concatenate(numpy_est))))
    ihvp = [b/scale for b in cur_estimate]
    ihvp = torch.squeeze(torch.stack(ihvp))
    ihvp = [a/num_samples for a in ihvp]
    ihvp = torch.squeeze(torch.stack(ihvp))

# print(ihvp)

eqn_2 = np.array([])
eqn_5 = np.array([])

ihvp = ihvp.detach()
print_iter = int(len(x_train)/100)
print_cntr = 0
for i in range(len(x_train)):
    x = x_train[i]
    x.requires_grad = True
    x_out = model(x.view(1,-1))
    x_loss = criterion(x_out,y_train[i].reshape(1))
    x_loss.backward(create_graph=True)
    grads = model.linear_3.weight.grad
    grads = grads.squeeze()

    infl = (torch.dot(ihvp.view(1,-1).squeeze(),grads.view(-1,1).squeeze())/len(x_train))
    i_pert = grad(infl,x)
    i_pert = i_pert[0]

    eqn_2 = np.vstack((eqn_2,-infl.detach().cpu().numpy())) if eqn_2.size else -infl.detach().cpu().numpy()
    eqn_5 = np.vstack((eqn_5,-i_pert.detach().cpu().numpy())) if eqn_5.size else -i_pert.detach().cpu().numpy()
    model.zero_grad()
    if (i % print_iter ==0) or (i==len(x_train)-1):
        print("Done "+str(print_cntr)+"/100")
        print_cntr +=1

np.save('results/eqn_2-test_set_smote.npy',eqn_2)
np.save('results/eqn_5-test_set_smote.npy',eqn_5)

elapsed_time = time.time()-start_time
# np.save('results/mlp_influence_time',elapsed_time)


import operator
import unittest
import json
from server import nlp
import spacy

class TestNLP(unittest.TestCase):

    @classmethod
    def setUpClass(cls):
        cls.nlp = nlp.nlp("../models/madlib_5cat_model")

    @classmethod
    def tearDownClass(cls):
        cls.nlp = None

    def test_pos_tagging(self):
        print("test_pos_tagging")
        test_sent = 'Tom Cruise jumps on couches.'

        d = json.loads(self.nlp.pos_tag(test_sent))

        self.assertEqual(d['pos'][0],'NNP')
        self.assertEqual(d['pos'][1], 'NNP')
        self.assertEqual(d['pos'][2], 'VBZ')
        self.assertEqual(d['pos'][4], 'NNS')

    def test_classify(self):
        print("test_classify")
        test_sports = 'The NFL stands for the National Football League. The NFL plays footballs on Sunday where people throw, kick, and run the ball for touchdowns and field goals.'
        cat = json.loads(self.nlp.classify_doc(test_sports))
        maxcategory = max(cat.items(), key=operator.itemgetter(1))[0]
        self.assertEqual(maxcategory,'sports')

        test_politics = 'The House and Senate today argued extensively over something pointless. Mitch McConnell yelled at Chuck Schumer. Congress is not expected to pass any new laws.'
        cat = json.loads(self.nlp.classify_doc(test_politics))
        maxcategory = max(cat.items(), key=operator.itemgetter(1))[0]
        self.assertEqual(maxcategory,'politics')

        test_tech = 'Microsoft released a new piece of software that is for playing video games and compiling code. Silicon Valley is full of technology'
        cat = json.loads(self.nlp.classify_doc(test_tech))
        maxcategory = max(cat.items(), key=operator.itemgetter(1))[0]
        self.assertEqual(maxcategory,'tech')

    def test_pos_to_tokens(self):
        print("test_doc")
        test_doc = "This is a test sentence document"
        nlpdoc = self.nlp.doc_to_pos_tokens(test_doc)
        self.assertEqual(isinstance(nlpdoc,spacy.tokens.doc.Doc),True)

    def test_pos_tokens_to_json(self):
        print("test_pos_tokens_to_json")
        test_doc = "This is a test sentence document"
        nlpdoc = self.nlp.doc_to_pos_tokens(test_doc)
        jsontext = self.nlp.pos_tokens_to_json(nlpdoc)
        self.assertIsNotNone(jsontext)
        dict = json.loads(jsontext)
        self.assertIsNotNone(dict['words'])
        self.assertIsNotNone(dict['pos'])

if __name__ == '__main__':
    unittest.main()


f=open("random","r")
words=[]
for lines in f:
    #remove \n
    line=lines.rstrip("\n")
    #splict by space
    word=line.split(" ")

    for w in word:
        #convert to uppercase
        words.append(w.upper())
        #convert to lowercase
        #words.append(w.lower())
print(words)

File Input & Output Function/upper and lower case.py


import torch
import torch.nn as nn
import scipy

class AttentionWalkLayer(nn.Module):
    def __init__(self, n_nodes, emb_dim, window_size, n_walks, beta, gamma, attention, normalize, temperature, shared):
        super(AttentionWalkLayer, self).__init__()
        self.left_emb = nn.Parameter(torch.zeros((n_nodes, emb_dim//2)), requires_grad=True)
        if shared:
            self.right_emb = self.left_emb
        else:
            self.right_emb = nn.Parameter(torch.zeros((n_nodes, emb_dim//2)), requires_grad=True)
        self.attention_method = attention
        self.normalize_method = normalize
        if self.normalize_method == 'sum':
            if self.attention_method not in ('global_exponential', 'personalized_exponential', 'global_gamma', 'personalized_gamma'):
                print('Sum normalize method require all entries are positive')
                exit()
        self.attention = None
        self.q = None
        self.k = None
        self.theta = None
        self.a = None
        self.b = None
        self.c = None
        self.d = None
        self.temperature = temperature

        if attention == 'constant':
            self.attention = nn.Parameter(torch.ones(window_size), requires_grad=False)
        elif attention == 'global_vector':
            self.attention = nn.Parameter(torch.ones(window_size), requires_grad=True)
        elif attention == 'global_exponential':
            self.q = nn.Parameter(torch.zeros(1), requires_grad=True)
        elif attention == 'global_gamma':
            self.k = nn.Parameter(torch.ones(1), requires_grad=True)
            self.theta = nn.Parameter(torch.ones(1), requires_grad=True)
        elif attention == 'global_linear':
            self.q = nn.Parameter(torch.ones(1), requires_grad=True)
        elif attention == 'global_quadratic':
            self.a = nn.Parameter(torch.tensor([-1.]), requires_grad=True)
            self.b = nn.Parameter(torch.ones(1), requires_grad=True)
            self.c = nn.Parameter(torch.ones(1), requires_grad=True)
        elif attention == 'global_cubic':
            self.a = nn.Parameter(torch.tensor([-1.]), requires_grad=True)
            self.b = nn.Parameter(torch.ones(1), requires_grad=True)
            self.c = nn.Parameter(torch.ones(1), requires_grad=True)
            self.d = nn.Parameter(torch.ones(1), requires_grad=True)
        elif attention == 'personalized_vector':
            self.attention = nn.Parameter(torch.ones((window_size, n_nodes)), requires_grad=True)
        elif attention == 'personalized_exponential':
            self.q = nn.Parameter(torch.zeros(n_nodes), requires_grad=True)
        elif attention == 'personalized_linear':
            self.q = nn.Parameter(torch.ones(n_nodes), requires_grad=True)
        elif attention == 'personalized_gamma':
            self.k = nn.Parameter(torch.ones(n_nodes), requires_grad=True)
            self.theta = nn.Parameter(torch.ones(n_nodes), requires_grad=True)
        elif attention == 'personalized_quadratic':
            self.a = nn.Parameter(torch.tensor([-1.]*n_nodes), requires_grad=True)
            self.b = nn.Parameter(torch.ones(n_nodes), requires_grad=True)
            self.c = nn.Parameter(torch.ones(n_nodes), requires_grad=True)
        elif attention == 'personalized_cubic':
            self.a = nn.Parameter(torch.tensor([-1.]*n_nodes), requires_grad=True)
            self.b = nn.Parameter(torch.ones(n_nodes), requires_grad=True)
            self.c = nn.Parameter(torch.ones(n_nodes), requires_grad=True)
            self.d = nn.Parameter(torch.ones(n_nodes), requires_grad=True)
        elif attention == 'personalized_function':
            self.linear = nn.Linear(emb_dim//2, window_size)
            nn.init.zeros_(self.linear.bias)
        else:
            print('Unexpected attention method')
            exit()

        self.n_nodes = n_nodes
        # self.transit_mat_series = transit_mat_series
        self.window_size = window_size
        self.n_walks = n_walks
        self.beta = beta
        self.gamma = gamma
        self.initialize_weights()

    def initialize_weights(self):
        nn.init.uniform_(self.left_emb, -0.1, 0.1)
        nn.init.uniform_(self.right_emb, -0.1, 0.1)
        # nn.init.uniform_(self.attention, -0.01, 0.01)

    def forward(self, adj_mat, transit_mat, degree):
        self.update_attention()

        if self.normalize_method == 'softmax':
            attention_probs = nn.functional.softmax(self.attention * self.temperature, dim=0)  # C
        elif self.normalize_method == 'sum':
            attention_probs = self.attention / (torch.sum(self.attention, dim=0))
        else:
            print('Unexpected normalize method')
            exit()

        transit_mat_power_n = torch.diag(torch.ones(self.n_nodes, dtype=torch.float, device=transit_mat.device))
        weighted_transit_mat = torch.zeros((self.n_nodes, self.n_nodes), dtype=torch.float, device=transit_mat.device)  # VxV
        for i in range(self.window_size):
            transit_mat_power_n = torch.mm(transit_mat_power_n, transit_mat)
            weighted_transit_mat += attention_probs[i] * transit_mat_power_n
        weighted_transit_mat *= degree * self.n_walks * self.n_nodes

        left_dot_right = torch.mm(self.left_emb, self.right_emb.transpose(0, 1))

        loss_on_target = -weighted_transit_mat.t() * nn.functional.logsigmoid(left_dot_right) # logsigmoid() is more numerically stable
        loss_on_opposite = -(1-adj_mat) * (-left_dot_right + nn.functional.logsigmoid(left_dot_right))  # log(1-sigmoid(x)) = -x + logsigmoid(x)
        loss_on_matrices = torch.mean(torch.abs(loss_on_target+loss_on_opposite))
        loss_on_regularization = self.beta * torch.mean(self.attention**2) \
                                 + self.gamma * (torch.mean(self.left_emb**2) + torch.mean(self.right_emb**2))

        if self.q is not None:
            loss_on_regularization += self.gamma * torch.mean(self.q)**2
        if self.k is not None:
            loss_on_regularization += self.gamma * torch.mean(self.k)**2
        if self.theta is not None:
            loss_on_regularization += self.gamma * torch.mean(self.theta)**2
        if self.a is not None:
            loss_on_regularization += self.gamma * torch.mean(self.a)**2
        if self.b is not None:
            loss_on_regularization += self.gamma * torch.mean(self.b)**2
        if self.c is not None:
            loss_on_regularization += self.gamma * torch.mean(self.c)**2
        if self.d is not None:
            loss_on_regularization += self.gamma * torch.mean(self.d)**2

        return loss_on_matrices + loss_on_regularization

    def update_attention(self):
        if self.attention_method in ('global_exponential', 'personalized_exponential'):
            q_abs = 2 * torch.sigmoid(self.q)
            mults = []
            for i in range(self.window_size):
                mults.append(0.99 * (q_abs ** i) + 0.01)
            self.attention = torch.stack(mults)
        elif self.attention_method in ('global_linear', 'personalized_linear'):
            q_neg = -1 * self.q
            mults = []
            for i in range(self.window_size):
                mults.append(q_neg * i)
            self.attention = torch.stack(mults)
        elif self.attention_method in ['global_gamma', 'personalized_gamma']:
            mults = []
            # k = 10*torch.sigmoid(self.k)
            # theta = 2*torch.sigmoid(self.theta)
            for i in range(1, self.window_size+1):
                mults.append(torch.pow(i, self.k-1)*torch.exp(-self.theta*i))
            self.attention = torch.stack(mults)
        elif self.attention_method in ['global_quadratic', 'personalized_quadratic']:
            mults = []
            for i in range(1, self.window_size+1):
                mults.append(self.a*i**2 + self.b*i + self.c)
            self.attention = torch.stack(mults)
        elif self.attention_method in ['global_cubic', 'personalized_cubic']:
            mults = []
            for i in range(1, self.window_size+1):
                mults.append(self.a*i**3 + self.b*i**2 + self.c*i + self.d)
            self.attention = torch.stack(mults)
        elif self.attention_method == 'personalized_function':
            self.attention = torch.t(self.linear(self.left_emb))  # n_nodes*window_size --> window_size*n_nodes


#!/usr/bin/env python

from common import load_root_node, natural_sort

# Python2 compatibility
try:
    raw_input
except NameError:
    raw_input = input

def choose(options, allow_multi):
    reverse_map = {}
    for i, (key, value) in enumerate(options):
        print("%3d) %s" % (i+1, key))
        reverse_map[i+1] = value
    print("  0) Back")
    while True:
        try:
            values = list(map(int, raw_input("Choose> ").split()))
            if len(values) == 0:
                continue
            if 0 in values:
                return
            values = [reverse_map[value] for value in values if value in reverse_map]
            if allow_multi:
                return values
            else:
                if len(values) == 1:
                    return values[0]
        except (ValueError, IndexError):
            print("Invalid input, please try again")
            pass

def main():
    node = load_root_node()

    while True:
        options = []
        will_download = True
        for n in node.get_children():
            options.append((n.title, n))
            if not n.can_download:
                will_download = False
        options = natural_sort(options, key=lambda x: x[0])
        result = choose(options, allow_multi=will_download)
        if result is None:
            if node.parent is not None:
                node = node.parent
            else:
                break
        elif will_download:
            for n in result:
                if not n.download():
                    raw_input("Press return to continue...\n")
        else:
            node = result

if __name__ == "__main__":
    try:
        main()
    except (KeyboardInterrupt, EOFError):
        print("\nExiting...")


def SelectionSort(val):
    for i in range(len(val)-1,0,-1):
        Max=0
        for l in range(1,i+1):
            if val[l]>val[Max]:
                Max = l
        temp = val[i]
        val[i] = val[Max]
        val[Max] = temp

Angka = [22,10,15,3,8,2]
SelectionSort(Angka)
print(Angka)


#Legacy base classes for Molecular Biology
import re
import os
import subprocess
from Bio import SeqIO, AlignIO
from Bio.Blast import NCBIXML
import sys

BLAST_PATH = ""

#delegate decorator
def delegate(method, prop):
    def decorate(cls):
        setattr(cls, method,
            lambda self, *args, **kwargs:
            getattr(getattr(self, prop), method)(*args, **kwargs))
        return cls
    return decorate

#DNA superclass
@delegate('__len__', 'sequence')
class DNA(object):

    complement_alphabet = {'A':'T', 'T':'A', 'C':'G', 'G':'C'}

    def __init__(self, number=None, description= None, sequence='', **kwargs):
        self.number = number
        self.description = description
        self.sequence = sequence.upper()
        self.circular = None
        self.warnings = [] #will be used to handle any non-lethal warnings, use dict or list?

    def reverse_complement(self):
        rev_comp_dna = "".join([self.complement_alphabet.get(nucleotide, '') for nucleotide in self.sequence[::-1]])
        return DNA(sequence = rev_comp_dna)

    def simulate_PCR(self, oligo1, oligo2, base_tm=50, max_length=60):
        high_gc_cutoff = 0.65
        matches = []
        for x in [oligo1, oligo2]:
            #get binding region of oligos
            annealing_seq =    DNA.get_seq_by_tm(x.sequence, base_tm, max_length, position = "right")
            x.tail_seq = x.sequence[:-len(annealing_seq)]
            fwd_matches = [m.start() for m in re.finditer(annealing_seq, self.sequence)]
            revcomp = self.reverse_complement()
            rev_matches =    [len(self.sequence)-m.start() for m in re.finditer(annealing_seq, revcomp.sequence)]
            matches.append( (fwd_matches, rev_matches) )
        flatten1 = [item for sublist in matches for item in sublist]
        flattened = [item for sublist in flatten1 for item in sublist]
        if flattened < 2:
            self.warnings.append("Number of primer bindings was < 2")
        #check if heads of oligos point at each other and are <10kb, use the smallest possible amplicon, and note other possibilities in warnings
        shortest_amplicon_length = 9999
        shortest_amplicon = []
        amplicon_found = False
        fwd_oligo = None
        rev_oligo = None
        if matches[0][0] and matches[1][1]:
            (f, r) = matches[0][0][0], matches[1][1][0]
            if f < r and (r - f < shortest_amplicon_length and r-f > 0):
                #oligo1 fwd, oligo2 rev
                fwd_oligo = oligo1
                rev_oligo = oligo2
                shortest_amplicon_length = r - f
                shortest_amplicon = [f, r]
                amplicon_found = True
        if matches[1][0] and matches[0][1]:
            (f, r) = matches[1][0][0], matches[0][1][0]
            if f < r and (r - f < shortest_amplicon_length and r-f > 0):
            #oligo2 fwd, oligo1 rev
                fwd_oligo = oligo2
                rev_oligo = oligo1
                shortest_amplicon_length = r - f
                shortest_amplicon = [f, r]
                amplicon_found = True
        if not(amplicon_found):
            raise Exception("No amplicon less than 9999bp")
        amplicon_seq = fwd_oligo.tail_seq + self.sequence[min(shortest_amplicon):max(shortest_amplicon)] + DNA(sequence = rev_oligo.tail_seq).reverse_complement().sequence
        amplicon = DNA(sequence = amplicon_seq)
        #todo, warnings (low gc, high gc)
        if DNA.gc_content(amplicon_seq) > high_gc_cutoff:
            amplicon.warnings.append("Amplicon has high gc (> %s)" % high_gc_cutoff)
        return amplicon
    @staticmethod
    def gc_content(sequence):
        if len(sequence)==0:
            return 0.0
        else:
            sequence = sequence.upper()
            num_g = sequence.count('G')
            num_c = sequence.count('C')
            gc = float(num_g + num_c)/float(len(sequence))
            return gc
    @staticmethod
    def simple_tm(sequence):
        if len(sequence)==0:
            return 0.0
        else:
            return 64.9+41*(DNA.gc_content(sequence)*len(sequence) - 16.4)/len(sequence)
    @staticmethod
    def get_seq_by_tm( sequence, tm, max_length, position = "right"):
        if position == 'right':
            sequence = DNA(sequence = sequence).reverse_complement().sequence
        elif position == "left":
            sequence = sequence
        else:
            raise Exception("position must be 'left' or 'right'")
        if tm < 45 or tm > 85:
            raise Exception("tm parameter must be 45 < tm < 85")
        offset = 15
        temp_substring = sequence[:offset]
        while(DNA.simple_tm(temp_substring) < tm) and offset <= max_length:
            # Allow substring return even if Tm is low
            offset += 1
            temp_substring = sequence[:offset]
        if position == 'right':
            temp_substring = DNA(sequence = temp_substring).reverse_complement().sequence
        return temp_substring

class Oligo(DNA):
    def __init__(self, *args, **kwargs):
        super(Oligo, self).__init__(*args, **kwargs)
        self.circular = False
        # primer3 output attributes
        self.sequence = kwargs.pop('sequence', '')
        self.direction = kwargs.pop('direction', 'LEFT')
        self.tm = float(kwargs.pop('tm', DNA.simple_tm(self.sequence)))
        self.hairpin = float(kwargs.pop('hairpin', -1))
        self.selfcomp_3 = float(kwargs.pop('selfcomp_3', -1))
        self.selfcomp_any = float(kwargs.pop('selfcomp_any', -1))
    def __repr__(self):
        return '\t'.join(map(str,[self.number,
                                    self.description,
                                    self.sequence,
                                    'Tm='+str(round(self.tm,1)),
                                    'GC%='+str(round(100*DNA.gc_content(self.sequence),1))
                                    ]))+'\n'

class BlastHandler(object):
    def __init__(self, db_fasta = None):
        self.make_blast_db(db_fasta)
        self.db_name = db_fasta
        #todo, double all strings, we are assuming they are all circular plasmids

    def blast_helper(self, blast_command):
        result_handle = os.popen(str(blast_command))
        return NCBIXML.read(result_handle)
    def blast_perfect_matching_substrings(self, needle_file):
        return self.blast(needle_file, "-ungapped -perc_identity 100")
    def blast(self, needle_file, extra_params=None):
        blastn_command = BLAST_PATH + "blastn -query '"+needle_file+"' -db "+self.db_name+" -outfmt 5"
        if extra_params:
            blastn_command = blastn_command + " " + extra_params
        return self.blast_helper(blastn_command)
    def blast_str(self, query_str):
        blastn_command = ' '.join([BLAST_PATH+'blastn','-query','-','-db',self.db_name,'-outfmt','5','<<<',query_str])
        return self.blast_helper(blastn_command)
    def blastn_short_str(self, query_str):
        blastn_command = ' '.join([BLAST_PATH+'blastn','-task','blastn-short','-ungapped','-query','-','-db',self.db_name,'-outfmt','5','<<<',query_str])
        return self.blast_helper(blastn_command)
    def make_blast_db(self, db_fasta):
        makeblastdb_command = BLAST_PATH + "makeblastdb -in "+db_fasta+" -dbtype 'nucl'"
        return subprocess.call(makeblastdb_command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True)


import torch
import torch.distributed as dist

class BarlowTwinsLoss(torch.nn.Module):
    """Implementation of the Barlow Twins Loss from Barlow Twins[0] paper.
    This code specifically implements the Figure Algorithm 1 from [0].

    [0] Zbontar,J. et.al, 2021, Barlow Twins... https://arxiv.org/abs/2103.03230

        Examples:

        >>> # initialize loss function
        >>> loss_fn = BarlowTwinsLoss()
        >>>
        >>> # generate two random transforms of images
        >>> t0 = transforms(images)
        >>> t1 = transforms(images)
        >>>
        >>> # feed through SimSiam model
        >>> out0, out1 = model(t0, t1)
        >>>
        >>> # calculate loss
        >>> loss = loss_fn(out0, out1)

    """

    def __init__(self, lambda_param: float = 5e-3, gather_distributed: bool = False):
        """Lambda param configuration with default value like in [0]

        Args:
            lambda_param:
                Parameter for importance of redundancy reduction term.
                Defaults to 5e-3 [0].
            gather_distributed:
                If True then the cross-correlation matrices from all gpus are
                gathered and summed before the loss calculation.
        """
        super(BarlowTwinsLoss, self).__init__()
        self.lambda_param = lambda_param
        self.gather_distributed = gather_distributed

        if gather_distributed and not dist.is_available():
            raise ValueError(
                "gather_distributed is True but torch.distributed is not available. "
                "Please set gather_distributed=False or install a torch version with "
                "distributed support."
            )

    def forward(self, z_a: torch.Tensor, z_b: torch.Tensor) -> torch.Tensor:
        device = z_a.device

        # normalize repr. along the batch dimension
        z_a_norm = (z_a - z_a.mean(0)) / z_a.std(0)  # NxD
        z_b_norm = (z_b - z_b.mean(0)) / z_b.std(0)  # NxD

        N = z_a.size(0)
        D = z_a.size(1)

        # cross-correlation matrix
        c = torch.mm(z_a_norm.T, z_b_norm) / N  # DxD

        # sum cross-correlation matrix between multiple gpus
        if self.gather_distributed and dist.is_initialized():
            world_size = dist.get_world_size()
            if world_size > 1:
                c = c / world_size
                dist.all_reduce(c)

        # loss
        c_diff = (c - torch.eye(D, device=device)).pow(2)  # DxD
        # multiply off-diagonal elems of c_diff by lambda
        c_diff[~torch.eye(D, dtype=bool)] *= self.lambda_param
        loss = c_diff.sum()

        return loss


"""
Modified for Ripple neural recording system. To load the dataset, we
use io.BlackrockIO.

@author: yc25258
"""
import numpy as np
import scipy as sp
from neo import io

def create_syncHDF(filename, Ripple_folder):
    #Ripple_folder = 'F:/yc25258/Ripple_example_data/Example12/'
    #filename = 'datafile0012'
    data_filename = Ripple_folder + filename + ".nev"

    # Load Ripple files.
    r = io.BlackrockIO(data_filename)
    bl = r.read_block()
    print("File read.")

    # Analogsignals for digital events
    # Naming convention
    # 0 - 3  : SMA 1 - 4
    # 4 - 27 : Pin 1 - 24
    # 28 - 29: Audio 1 - 2
    # Here we use Pin 1, 2, 3, 4, 5, 7, 8, 9, 10, 12, 13, 14, 15, 17, 18, 19
    signals = bl.segments[0].analogsignals[-1]
    pins_util = np.array([1, 2, 3, 4, 5, 7, 8, 9, 10, 12, 13, 14, 15, 17, 18, 19]) + 3
    rstart = np.array(signals[:, 22 + 3])
    strobe = np.array(signals[:, 20 + 3])
    msgtype = np.array(signals[:, pins_util[8:]])
    rwoNum = np.array(signals[:, pins_util[:8]])
    # Convert to binary numbers (0 ~ 5000 from the recordings)
    msgtype[msgtype<=2500] = 0
    msgtype[msgtype>2500] = 1
    rwoNum[rwoNum<=2500] = 0
    rwoNum[rwoNum>2500] = 1
    rstart[rstart<=2500] = 0
    rstart[rstart>2500] = 1
    strobe[strobe<=2500] = 0
    strobe[strobe>2500] = 1
    rstart = np.ravel(rstart)
    strobe = np.ravel(strobe)
    # Convert the binary digits into arrays
    MSGTYPE = np.zeros(strobe.shape)
    ROWNUMB = np.zeros(strobe.shape)
    for tp in range(MSGTYPE.shape[0]):
        temp_str = ''
        for i in range(8):
            temp_str += str(int(msgtype[tp, 7-i]))
        MSGTYPE[tp] = int(temp_str, 2)
        temp_str = ''
        for i in range(8):
            temp_str += str(int(rwoNum[tp, 7-i]))
        ROWNUMB[tp] = int(temp_str, 2)
    print("Row Number extracted.")

    # Create dictionary to store synchronization data
    hdf_times = dict()
    hdf_times['row_number'] = []          # PyTable row number
    hdf_times['ripple_samplenumber'] = []    # Corresponding Ripple sample number
    hdf_times['ripple_dio_samplerate'] = []  # Sampling frequency of DIO signal recorded by Ripple system
    hdf_times['ripple_recording_start'] = [] # Ripple sample number when behavior recording begins

    find_recording_start = np.ravel(np.nonzero(rstart))[0]
    find_data_rows = np.logical_and(np.ravel(np.equal(MSGTYPE,13)),np.ravel(np.greater(strobe,0)))
    find_data_rows_ind = np.ravel(np.nonzero(find_data_rows))

    rows = ROWNUMB[find_data_rows_ind]	  # row numbers (mod 256)

    prev_row = rows[0] 	# placeholder variable for previous row number
    counter = 0 		# counter for number of cycles (i.e. number of times we wrap around from 255 to 0) in hdf row numbers

    for ind in range(1,len(rows)):
    	row = rows[ind]
    	cycle = (row < prev_row) # row counter has cycled when the current row number is less than the previous
    	counter += cycle
    	rows[ind] = counter*256 + row
    	prev_row = row

    # Load data into dictionary
    hdf_times['row_number'] = rows
    hdf_times['ripple_samplenumber'] = find_data_rows_ind
    hdf_times['ripple_recording_start'] = find_recording_start
    hdf_times['ripple_dio_samplerate'] = signals.sampling_rate

    # Save syncing data as .mat file
    mat_filename = filename + '_syncHDF.mat'
    #sp.io.savemat(Ripple_folder+mat_filename,hdf_times)
    print(Ripple_folder+mat_filename)

    return hdf_times


import numpy as np
import matplotlib.pyplot as plt

font1 = {'family': 'Times New Roman',
         'weight': 'normal',
         'size': 25,
         }

font2 = {'family': 'Times New Roman',
         'weight': 'normal',
         'size': 35,
         }

fig = plt.figure(figsize=(10, 10))
sub = fig.add_subplot(111)

shiftY = 0.4
shiftX = 35
ms = 11

fontsize1 = 28
fontsize2 = 35

model_steps_30 = [300, 480]
model_score_30 = [69.89, 73.24]
l2, = sub.plot(model_steps_30, model_score_30, 'b^-', ms=ms)
sub.text(model_steps_30[-1] + shiftX, model_score_30[-1], "300k\nTask", ha='center', va='center', fontsize=fontsize1)

rand_steps_30 = [300, 480]
rand_score_30 = [69.89, 72.20]
l3, = sub.plot(rand_steps_30, rand_score_30, 'gs-', ms=ms)
sub.text(rand_steps_30[-1], rand_score_30[-1] + shiftY, "300k\nRand.", ha='center', va='center', fontsize=fontsize1)

model_steps_20 = [200, 366]
model_score_20 = [69.14, 73.02]
sub.plot(model_steps_20, model_score_20, 'b^-', ms=ms)
sub.text(model_steps_20[-1], model_score_20[-1] + shiftY, "200k\nTask", ha='center', va='center', fontsize=fontsize1)

rand_steps_20 = [200, 384]
rand_score_20 = [69.14, 72.04]
sub.plot(rand_steps_20, rand_score_20, 'gs-', ms=ms)
sub.text(rand_steps_20[-1] + shiftX * 4 / 3, rand_score_20[-1], "200k\nRand.", ha='center', va='center', fontsize=fontsize1)

model_steps_10 = [100, 252]
model_score_10 = [68.78, 72.98]
sub.plot(model_steps_10, model_score_10, 'b^-', ms=ms)
sub.text(model_steps_10[-1], model_score_10[-1] + shiftY, "100k\nTask", ha='center', va='center', fontsize=fontsize1)

rand_steps_10 = [100, 244]
rand_score_10 = [68.78, 71.78]
sub.plot(rand_steps_10, rand_score_10, 'gs-', ms=ms)
sub.text(rand_steps_10[-1] + shiftX, rand_score_10[-1] - shiftY, "100k\nRand.", ha='center', va='center', fontsize=fontsize1)

main_steps = [100, 200, 300]
main_score = [68.78, 69.14, 69.89]
l1, = sub.plot(main_steps, main_score, 'ro-', ms=ms)

for step, score in zip(main_steps, main_score):
    sub.text(step + shiftX, score - shiftY / 2, str(step) + "k", ha='center', va='center', fontsize=fontsize1)
sub.hlines(72.57, 100, 500, colors="gray", linestyles="dashed", linewidth=3)
sub.text(210, 72.70, "Fully-trained (1M steps)", ha='center', va='center', fontsize=27)
sub.text(48, 72.6, "72.6-", ha='center', va='center', fontsize=fontsize2)

plt.grid()
plt.tick_params(labelsize=fontsize2)

plt.xlabel("k Steps", font2)
plt.ylabel("Acc.(%)", font2)

plt.legend(handles=[l1, l2, l3], labels=['General Pre-train', 'Selective Mask', 'Random Mask'], loc='lower right', prop=font1)
plt.savefig("../images/lap_amazon.pdf", foramt="pdf")


#!/usr/bin/env python
import sys,re
#git format-patch --numbered-files --start-number=10001
#using start number is 10001
#
#The result is in "commit_patch.txt". Format like:
#10001 b352dde
#10002 b310792
#10003 38e5a5a
#10004 2509814
#10005 5b782e3
#...
num = 10001;
print("opened file is commit_patch.txt\n")
inputnum = input("Enter input number(should > 10001):")
f = open("commit_patch.txt", "w+")#only save one line!!
for num in range(10001, inputnum):
	try:
		in_file = open(str(num))
		line = in_file.readline()     #just read the first line
		list = re.findall(r'(.{4})',line)
		if list[0] == "From":
			f.write(str(num) + list[1] + list[2]+"\n")
		in_file.close()
	except:
		#pass
		print("file " + str(num) + " not exits")

#def search(str, list)
#	for i in range(1, len(list))
#		if list[i] == str:
#			id = list[i-1]
#		i = i + 4
#	return id