# -*- coding: utf-8 -*-
"""
Created on Sun Mar  1 21:34:35 2020

@author: KAVITA
"""

from moviepy.editor import *
import sys
import matplotlib.pyplot as plt
import scipy
from scipy import signal
from scipy.io import wavfile
import librosa as lib
scipy.signal.spectrogram
import numpy as np
import librosa.display
import pickle as pkl

def get_audio_features(source_filepath,destination_filepath,i,label):

    feature_files = ".\Data\enterface_features_encoded"
    video = VideoFileClip(source_filepath)
    audio_part = video.audio
    audio_part.write_audiofile(destination_filepath)

    sample_rate, samples = wavfile.read(destination_filepath)
    frequencies, times, spectrogram = signal.spectrogram(samples, sample_rate)

    ##### read audio as vector

    audio_input,sr=lib.load(destination_filepath)

    ##### Hamming Window and stft
    audio_stft = librosa.stft(audio_input,window='hamm',n_fft=1024)
    audio_mfcc=librosa.feature.mfcc(abs(audio_input),n_mfcc=32)

    output_filename = os.path.join(feature_files, str(i) + '.pkl')
    out = {'class_label': label,
       'audio': audio_mfcc,
       'sr': sr}
    with open(output_filename, 'wb') as w:
        pkl.dump(out, w)

def get_einterface_files():
    count_unknown, i = 0,0
    av_file_directory = ".\Data\eInterface/enterface database"
    aud_folder='.\Data\Audio_clips\enterface'
    emotions = ['anger', 'disgust','fear', 'happiness', 'sadness', 'surprise']
    for (root,dirs,files) in os.walk(".\Data\eInterface/enterface database", topdown=True):
        for file in files:
            if str(file) != "Thumbs.db": #I put this condition because on my Mac I had a bunch of thumbnail files which we want to skip
                if not str(file).startswith('.'):
                    source_path = os.path.join(root,file)
                    print(source_path)
                    if str(source_path).find("fear") > 0:
                        aud_file_name='f{}'.format(i)+'.wav'
                        label="2"

                    elif str(source_path).find("happiness") > 0 :
                        aud_file_name='h{}'.format(i)+'.wav'
                        label="3"

                    elif str(source_path).find("sadness") > 0:
                        #print(str(root).find("sadness"))
                        aud_file_name='sa{}'.format(i)+'.wav'
                        label="4"

                    elif str(str(source_path)).find("surprise") > 0:
                        aud_file_name='su{}'.format(i)+'.wav'
                        label="5"

                    elif str(str(source_path)).find("anger") > 0 :
                        aud_file_name='a{}'.format(i)+'.wav'
                        label="0"

                    elif str(source_path).find("disgust") > 0 :
                        aud_file_name='d{}'.format(i)+'.wav'
                        label="1"

                    else :
                        #print(str(source_path))
                        aud_file_name = "unknown"
                        count_unknown +=1

                    if aud_file_name != "unknown":
                        #print(vid_file_name)
                        #pass
                        get_audio_features(source_path,os.path.join(aud_folder,aud_file_name),i,label)
                        i+=1

if __name__ == "__main__":

    #get_savee_files()
    get_einterface_files()
    #python open_enterface_folders.py


import pytest
from flask import Flask

from useraccounts.app import flask_app

@pytest.fixture(autouse=True)
def fake_bootstrap(monkeypatch):
    monkeypatch.setattr('useraccounts.app.bootstrap', lambda: None)
    monkeypatch.setattr('useraccounts.app.register_blueprints', lambda *args, **kwargs: None)

def test_create_app_should_return_instance_of_flask_app():
    app = flask_app()
    assert isinstance(app, Flask)

# def test_create_app_should_abort_on_missing_config(monkeypatch):
#     def mock_abort(x):
#         raise Exception('Start up aborted!')

#     monkeypatch.setitem('useraccounts.bootstrap.abort', mock_abort)
#     monkeypatch.setitem('useraccounts.config.config', 'DB_CONNECTION_STRING', None)

#     with pytest.raises(Exception) as excinfo:
#         flask_app()

#     assert excinfo.value.args[0] == 'Start up aborted!'


from django.shortcuts import render

# Create your views here.
import os
import time
from mysite import settings

def show_static(request):
    print(request.META['REMOTE_ADDR'])  # 客户端ｉｐ
    return render(request, 'index.html')

def show_qr(request):
    remote_addr = request.META['REMOTE_ADDR']
    print(remote_addr)

    img_path = 'static//%s.png' % remote_addr
    if os.path.exists(img_path):
        os.remove(img_path)
    app_dir = 'wechat//wechatHelper'
    img_path = '..//..//' + img_path  # 这个是要传入的参数，因为在WeChatHelper.py文件中要切换目录，所以这里要向上跳两个目录级别
    cmd = 'python3 wechat/wechatHelper/WeChatHelper.py --base_dir %s --pro_dir %s --img_path %s &' \
          % (settings.BASE_DIR, app_dir, img_path)
    os.system(cmd)
    time.sleep(3)  # 这里必须要给时间去下载二维码
    context = {}
    context['url'] = '/static/%s.png' % remote_addr
    return render(request, 'showqr.html', context)


import streamlit
import pandas
import snowflake.connector

streamlit.title('My First Hotel Menu by Streamlit')
#streamlit.header('Breakfast Menu')
streamlit.header("Fruityvice Fruit Advice!")
streamlit.text('Omega 3 & Blueberry Oatmeal')
streamlit.text('Kale, Spinach & Rocket Smoothie')
streamlit.text('Hard-Boiled Free-Range Egg')

my_fruit_list = pandas.read_csv("https://uni-lab-files.s3.us-west-2.amazonaws.com/dabw/fruit_macros.txt")
my_fruit_list = my_fruit_list.set_index('Fruit')

fruits_selected =streamlit.multiselect("Pick some fruits:", list(my_fruit_list.index),['Avocado','Strawberries'])
streamlit.dataframe(my_fruit_list)

fruits_to_show = my_fruit_list.loc[fruits_selected]

import requests
fruityvice_response = requests.get("https://fruityvice.com/api/fruit/" + "watermelon")
streamlit.text(fruityvice_response)
# write your own comment -what does the next line do?
fruityvice_normalized = pandas.json_normalize(fruityvice_response.json())
# write your own comment - what does this do?
streamlit.dataframe(fruityvice_normalized)

my_cnx = snowflake.connector.connect(**streamlit.secrets["snowflake"])
my_cur = my_cnx.cursor()
my_cur.execute("SELECT * from pc_rivery_db.public.fruit_load_list")
my_data_row = my_cur.fetchone()
streamlit.text("The fruit list contains:")
streamlit.text(my_data_row)


import pygame
from checkers.constants import WIDTH, HEIGHT, BOXSIZE, BROWN, WHITE, MAINMENU
from checkers.gamepiece import GamePiece
from checkers.gameLogic import GameLogic
from minimax import minimaxAlgo, getMoveList
import random

FRAMES_PER_SEC = 60

# Receive imported constants and set the width and height
WINDOW = pygame.display.set_mode((WIDTH, HEIGHT))
pygame.display.set_caption('Welcome to our checkers gameLogic!')

def useMouse(location):
    x, y = location
    row = int(y // BOXSIZE)
    column = int(x // BOXSIZE)
    return row, column

#displays options when game has ended
def showGameOverScreen(winner):
    font = pygame.font.SysFont('comicsansms', 36)
    if winner == BROWN:
        text = font.render("GAME OVER !!", True, WHITE)
        text2 = font.render("Brown player wins!", True, WHITE)
    elif winner == WHITE:
        text = font.render("GAME OVER !!", True, WHITE)
        text2 = font.render("White player wins!", True, WHITE)
    else:
        text = font.render("GAME OVER !!", True, WHITE)
        text2 = font.render("It's a draw!", True, WHITE)

    play_again_text = font.render("Play Again", True, WHITE)
    quit_text = font.render("Quit", True, WHITE)

    text_rect = text.get_rect(center=(WIDTH // 2, HEIGHT // 2 - 50))
    text2_rect = text2.get_rect(center=(WIDTH // 2, HEIGHT // 2))
    play_again_rect = play_again_text.get_rect(center=(WIDTH // 2, HEIGHT // 2 + 50))
    quit_rect = quit_text.get_rect(center=(WIDTH // 2, HEIGHT // 2 + 100))

    WINDOW.fill((0, 0, 0))
    WINDOW.blit(text, text_rect)
    WINDOW.blit(text2, text2_rect)
    pygame.draw.rect(WINDOW, (0, 0, 255), play_again_rect, border_radius=5)
    pygame.draw.rect(WINDOW, (0, 0, 255), quit_rect, border_radius=5)
    WINDOW.blit(play_again_text, play_again_rect)
    WINDOW.blit(quit_text, quit_rect)
    pygame.display.update()

#gameLoop for advanced mode
def twoPlayer():
    run = True
    clock = pygame.time.Clock()
    gameLogic = GameLogic(WINDOW)

    while run:
        clock.tick(FRAMES_PER_SEC)

        if gameLogic.getWinner() != None:
            print(gameLogic.getWinner())
            run = False

        for event in pygame.event.get():
            if event.type == pygame.QUIT:
                run = False

            if event.type == pygame.MOUSEBUTTONDOWN:
                location = pygame.mouse.get_pos()
                row, column = useMouse(location)
                gameLogic.choose(row, column)

        gameLogic.update()

    pygame.quit()

#game loop for hard mode
def hardMode():
    run = True
    clock = pygame.time.Clock()
    gameLogic = GameLogic(WINDOW)
    pygame.display.set_caption('Welcome to our checkers gameLogic -> hard mode')

    while run:
        clock.tick(FRAMES_PER_SEC)

        if gameLogic.turn == WHITE:
            v, newBoard = minimaxAlgo(gameLogic.getBoard(), 4, WHITE, gameLogic)
            gameLogic.aiMove(newBoard)

        if gameLogic.getWinner() != None:
            showGameOverScreen(gameLogic.getWinner())
            if playAgain() == True:
                run = playAgain()
                gameLogic.reset()
                mainMenu()

        for event in pygame.event.get():
            if event.type == pygame.QUIT:
                run = False

            if event.type == pygame.MOUSEBUTTONDOWN:
                location = pygame.mouse.get_pos()
                row, column = useMouse(location)
                gameLogic.choose(row, column)

        gameLogic.update()

    pygame.quit()

#game loop for easy mode
def easyMode():
    run = True
    clock = pygame.time.Clock()
    gameLogic = GameLogic(WINDOW)
    pygame.display.set_caption('Welcome to our checkers gameLogic -> easy mode')

    while run:
        clock.tick(FRAMES_PER_SEC)

        if gameLogic.turn == WHITE:
            moves = getMoveList(gameLogic.getBoard(), WHITE, gameLogic)
            if moves:
                move = random.choice(moves)
                gameLogic.aiMove(move)

        gameLogic.update()

        if gameLogic.getWinner() != None:
            showGameOverScreen(gameLogic.getWinner())
            if playAgain() == True:
                run = playAgain()
                gameLogic.reset()
                mainMenu()

        for event in pygame.event.get():
            if event.type == pygame.QUIT:
                run = False

            if event.type == pygame.MOUSEBUTTONDOWN:
                location = pygame.mouse.get_pos()
                row, column = useMouse(location)
                gameLogic.choose(row, column)

        gameLogic.update()

    pygame.quit()

#resart game funtion
def playAgain():
    while True:
        for event in pygame.event.get():
            if event.type == pygame.MOUSEBUTTONDOWN:
                location = pygame.mouse.get_pos()
                if WIDTH // 2 - 100 <= location[0] <= WIDTH // 2 + 100:
                    if HEIGHT // 2 + 50 <= location[1] <= HEIGHT // 2 + 100:
                        return True
                    elif HEIGHT // 2 + 100 <= location[1] <= HEIGHT // 2 + 150:
                        return False

#main selection menu
def mainMenu():
    pygame.init()

    run = True
    clock = pygame.time.Clock()

    while run:
        clock.tick(FRAMES_PER_SEC)
        WINDOW.fill((0, 0, 0))  # Fill the window with black color

        # Draw menu options
        font = pygame.font.SysFont('comicsansms', 36)

        intro = font.render("WELCOME TO OUR CHECKERS GAME", True, WHITE)
        easy_mode = font.render("1. Easy Mode", True, WHITE)
        start_text = font.render("2. Hard mode", True, WHITE)
        quit_text = font.render("Quit", True, WHITE)
        two_player = font.render("3. Multiplayer", True, WHITE)

        WINDOW.blit(MAINMENU, (0,0))
        WINDOW.blit(intro, (WIDTH // 2 - intro.get_width() // 2, HEIGHT // 2 - 200))
        WINDOW.blit(start_text, (WIDTH // 2 - start_text.get_width() // 2, HEIGHT // 2 + 50))
        WINDOW.blit(quit_text, (WIDTH // 2 - quit_text.get_width() // 2, HEIGHT // 2 + 250))
        WINDOW.blit(easy_mode, (WIDTH // 2 - easy_mode.get_width() // 2, HEIGHT // 2 - 50))
        WINDOW.blit(two_player, (WIDTH // 2 - easy_mode.get_width() // 2, HEIGHT // 2 + 150))

        pygame.display.update()

        for event in pygame.event.get():
            if event.type == pygame.QUIT:
                run = False

            if event.type == pygame.MOUSEBUTTONDOWN:
                location = pygame.mouse.get_pos()
                if WIDTH // 2 - start_text.get_width() // 2 <= location[0] <= WIDTH // 2 + start_text.get_width() // 2:
                    if HEIGHT // 2 - 50 <= location[1] <= HEIGHT // 2 - 50 + start_text.get_height():
                        hardMode()

                if WIDTH // 2 - easy_mode.get_width() // 2 <= location[0] <= WIDTH // 2 + easy_mode.get_width() // 2:
                    if HEIGHT // 2 + 50 <= location[1] <= HEIGHT // 2 + 50 + easy_mode.get_height():
                        easyMode()

                if WIDTH // 2 - two_player.get_width() // 2 <= location[0] <= WIDTH // 2 + two_player.get_width() // 2:
                    if HEIGHT // 2 + 150 <= location[1] <= HEIGHT // 2 + 150 + two_player.get_height():
                        twoPlayer()

                if WIDTH // 2 - quit_text.get_width() // 2 <= location[0] <= WIDTH // 2 + quit_text.get_width() // 2:
                    if HEIGHT // 2 + 250 <= location[1] <= HEIGHT // 2 + 250 + quit_text.get_height():
                        print("Thank you for playing!")
                        run = False

    pygame.quit()

mainMenu()


import pandas as pd
import numpy as np
from sklearn.model_selection import KFold
from sklearn.linear_model import LogisticRegression
from sklearn.tree import DecisionTreeClassifier
from sklearn.naive_bayes import GaussianNB
from sklearn.ensemble import RandomForestClassifier

train_data_file = pd.read_csv('data/traindata.csv')
train_labels_file = pd.read_csv('data/trainlabel.csv')
test_data_file = pd.read_csv('data/testdata.csv')

for i in train_data_file:
    train_data_file[i] = train_data_file[i] - np.mean(train_data_file[i])

attributes = train_data_file.values
labels = train_labels_file.values.ravel()
X_test = test_data_file.values

accuracy_mean = []
accuracy_ = []

log_cls = LogisticRegression()
dtree_clf = DecisionTreeClassifier(random_state=0)
gnb_clf = GaussianNB()
rf_clf = RandomForestClassifier(max_depth=2, random_state=0)

kf = KFold(n_splits=5, shuffle=False, random_state=None)
kf.get_n_splits(attributes)
for train_index, test_index in kf.split(attributes):
    X_train, X_test = attributes[train_index], attributes[test_index]
    y_train, y_test = labels[train_index], labels[test_index]

    # print(type(X_train))
    # print(X_train.shape)

    accuracy = []

    # using svm
    # svm_cls = svm.SVC(kernel='linear')
    # svm_cls.fit(X_train, y_train)
    # svm_predict = svm_cls.score(X_test, y_test)
    # accuracy.append(svm_predict)

    # using logistic regression
    log_cls.fit(X_train, y_train)
    log_accuracy = log_cls.score(X_test, y_test)
    accuracy.append(log_accuracy)

    # using decision tree
    dtree_clf.fit(X_train, y_train)
    dtree_accuracy = dtree_clf.score(X_test, y_test)
    accuracy.append(dtree_accuracy)

    # using gaussian naive bayes
    gnb_clf.fit(X_train, y_train)
    gnb_accuracy = gnb_clf.score(X_test, y_test)
    accuracy.append(gnb_accuracy)

    # using random forest
    rf_clf.fit(X_train, y_train)
    rf_accuracy = rf_clf.score(X_test, y_test)
    accuracy.append(rf_accuracy)

    accuracy_.append(accuracy)

for i in range(4):
    accuracy_mean.append((accuracy_[0][i] + accuracy_[1][i] + accuracy_[2][i] + accuracy_[3][i])/4)

print(accuracy_mean)

# as the first classifier gives the best performance in cross validation,
# therefore, it is chosen.

result = log_cls.predict(test_data_file)
result = pd.Series(result)
result.to_csv('project1_20446377.csv',index=False)


import numpy as np
import cv2
import torch
import torch.nn.functional as F

def smooth_mask(mask, size, sigma) -> np.ndarray:
    mask = mask.astype(np.uint8)
    mask = cv2.GaussianBlur(mask, (size, size), sigma)
    return mask

def dilate_mask(mask, kernel_size=3, iterations=1):
    kernel = np.ones((kernel_size, kernel_size), np.uint8)
    mask = cv2.dilate(mask, kernel, iterations=iterations)

    return mask

def erode_mask(mask, kernel_size=3, iterations=1):
    kernel = np.ones((kernel_size, kernel_size), np.uint8)
    mask = cv2.erode(mask, kernel, iterations=iterations)

    return mask

def close_small_holes(mask, min_size=10):
    contours, _ = cv2.findContours(mask, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)
    for contour in contours:
        if cv2.contourArea(contour) < min_size:
            cv2.drawContours(mask, [contour], -1, 0, -1)

    return mask

def maxpool2d_closing(image: np.ndarray, mask: np.ndarray):

    im_floodfill = mask.copy()

    h, w = image.shape[:2]
    flood_mask = np.zeros((h+2, w+2), np.uint8)

    ret, imf, maskf, rect = cv2.floodFill(im_floodfill, flood_mask, (0,0), 255)
    maskf = maskf[1:-1, 1:-1]

    img_conv = torch.tensor(image, dtype=torch.float32)
    img_conv = torch.permute(img_conv, (2, 0, 1)).unsqueeze(0)

    img_max = F.max_pool2d(img_conv, 3, 1, 1)
    img_max_numpy = img_max.permute(0, 2, 3, 1).numpy()[0]
    img_max_numpy = img_max_numpy.astype(np.uint8)

    mask_conv = 255 - mask
    mask_conv = mask_conv.reshape(1, 1, h, w)

    mask_max = F.max_pool2d(torch.tensor(mask_conv.astype(np.float32)), 3, 1, 1)
    mask_max_numpy = mask_max.numpy().astype(np.uint8)[0][0]
    mask_max_numpy = 255 - mask_max_numpy

    img_max_numpy[maskf == 1] = 0
    mask_max_numpy[maskf == 1] = 255


import serial

ser = serial.Serial()
ser.port = "/dev/ttyUSB1"
ser.baudrate = 115200
ser.bytesize = serial.EIGHTBITS
ser.parity = serial.PARITY_NONE
ser.timeout = 1.

ser.open()
print (ser.is_open)

while True:
    line = ser.read()
    print (line)

ser.close()
print (ser.is_open)


import requests
from common.classes.hugging_face_classes import HuggingFaceResponse

# retrieve tokens from .env file
from dotenv import load_dotenv
import os
load_dotenv()
HUGGING_FACE_TOKEN = os.getenv('HUGGING_FACE_TOKEN')

huggin_face_url = 'https://api-inference.huggingface.co/models/google/bigbird-pegasus-large-pubmed'

def summarizeWithML(abstracts_text: str):
  '''summarize a text using an ml model from Hugging Face'''
  hugging_face_response = requests.post(
    huggin_face_url,
    headers={
      "Authorization": f"Bearer {HUGGING_FACE_TOKEN}"
    },
    json=abstracts_text
  )
  hugging_face_obj: HuggingFaceResponse = hugging_face_response.json()
  return hugging_face_obj


# -*- coding: utf-8 -*-
"""
Created on Thu Nov 26 09:56:33 2020

@author: Administrator
"""

import torch
import torchvision as tv
import torchvision.transforms as transforms
import torch.nn as nn
import torch.optim as optim
from PIL import Image, ImageFile
ImageFile.LOAD_TRUNCATED_IMAGES = True
from torch.utils.data import Dataset, DataLoader
import argparse
import torchvision.models as models
import cv2
import time
import numpy as np
import matplotlib.pyplot as plt

device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
# device = torch.device("cuda")
parser = argparse.ArgumentParser()
parser.add_argument('--outf', default='F:/yzh/DeepLearningDataSet/model_16/', help='folder to output images and model checkpoints')
parser.add_argument('--net', default='/model/net.pth', help="path to netG (to continue training)")
opt = parser.parse_args()

# EPOCH = 30
BATCH_SIZE = 32     #(batch_size)
LR = 0.001        #learning rate
# root = 'D:/neck_tissue_datasets/'
dataset = 'F:/yzh/DeepLearningDataSet/'
# dataset = 'D:/NeckTissue/dataset1103/'
savepath = '/model_16/'
modelname = '1'

def default_loader(path):
    bgr = cv2.imread(path, cv2.IMREAD_LOAD_GDAL)
    rgb = bgr[:, :, ::-1]
    img = Image.fromarray(rgb).convert('L')

    # return Image.open(path).convert('L')
    return img
class MyDataset(Dataset):
    def __init__(self, txt, transform=None, target_transform=None, loader=default_loader):
        fh = open(txt, 'r')                 #Open the specified folder
        imgs = []
        for line in fh:
            line = line.strip('\n')
            line = line.rstrip()
            words = line.split()
            imgs.append((words[0],int(words[1])))
        self.imgs = imgs
        self.transform = transform
        self.target_transform = target_transform
        self.loader = loader

    def __getitem__(self, index):
        fn, label = self.imgs[index]
        img = self.loader(fn)
        if self.transform is not None:
            img = self.transform(img)
#        print(img.size())
        return img,label

    def __len__(self):
        return len(self.imgs)

#Reading data
transform1 = transforms.Compose([
                                 transforms.ToTensor(),

                                 # transforms.Grayscale(1)
                                 # transforms.Normalize((0.5,),(0.5,))
                                 ]
                                )
transform2 = transforms.ToTensor()
train_data1=MyDataset(txt=dataset+'train1.txt', transform=transform1)
train_data2=MyDataset(txt=dataset+'train2.txt', transform=transform1)
train_data3=MyDataset(txt=dataset+'train3.txt', transform=transform1)
train_data4=MyDataset(txt=dataset+'train4.txt', transform=transform1)
test_data1=MyDataset(txt=dataset+'test1.txt', transform=transform1)
test_data2=MyDataset(txt=dataset+'test2.txt', transform=transform1)
test_data3=MyDataset(txt=dataset+'test3.txt', transform=transform1)
test_data4=MyDataset(txt=dataset+'test4.txt', transform=transform1)

train_data=train_data2+test_data2+train_data3+test_data3+train_data4+test_data4
test_data=train_data1+test_data1

trainloader = DataLoader(dataset=train_data, batch_size=BATCH_SIZE, shuffle=True)
#testloader = DataLoader(dataset=test_data, batch_size=8)
testloader = DataLoader(dataset=test_data)

train_data_size = 40500  #Total training set (including all classes)
valid_data_size = 13500  #Total test set (including all classes)

class LeNet(nn.Module):
    def __init__(self):
        super(LeNet, self).__init__()
        self.conv1 = nn.Sequential(     #input_size=(1*28*28)
            nn.Conv2d(1, 6, 5, 1, 2),
            nn.ReLU(),      #input_size=(6*28*28)
            nn.MaxPool2d(kernel_size=2, stride=2),#output_size=(6*14*14)
        )
        self.conv2 = nn.Sequential(
            nn.Conv2d(6, 16, 5),
            nn.ReLU(),      #input_size=(16*10*10)
            nn.MaxPool2d(2, 2)  #output_size=(16*5*5)
        )
        self.conv3 = nn.Sequential(
            nn.Conv2d(16, 16, 5),
            nn.ReLU(),      #input_size=(16*10*10)
            nn.MaxPool2d(2, 2)  #output_size=(16*5*5)
        )
        self.fc1 = nn.Sequential(
            nn.Linear(16 * 29 * 29, 120),
            nn.ReLU()
        )
        self.fc2 = nn.Sequential(
            nn.Linear(120, 84),
            nn.ReLU()
        )
        self.fc3 = nn.Linear(84, 3)

    def forward(self, x):
        x = self.conv1(x)
        x = self.conv2(x)
        x = self.conv3(x)
        # nn.Linear()
        # print(x.size())
        x = x.view(x.size()[0], -1)
        x = self.fc1(x)
        x = self.fc2(x)
        x = self.fc3(x)
        return x

class LeNet12(nn.Module):
    def __init__(self):
        super(LeNet12, self).__init__()
        self.conv1 = nn.Sequential(     #input_size=(1*28*28)
            nn.Conv2d(1, 6, 5, 1, 2),
            nn.ReLU(),      #input_size=(6*28*28)
            nn.MaxPool2d(kernel_size=2, stride=2),#output_size=(6*14*14)
        )
        self.conv2 = nn.Sequential(
            nn.Conv2d(6, 16, 5, 1, 2),
            nn.ReLU(),      #input_size=(16*10*10)
            nn.MaxPool2d(2, 2)  #output_size=(16*5*5)
        )
        self.conv3 = nn.Sequential(
            nn.Conv2d(16, 32, 5, 1, 2),
            nn.ReLU(),      #input_size=(16*10*10)
            nn.MaxPool2d(2, 2)  #output_size=(16*5*5)
        )
        self.conv4 = nn.Sequential(
            nn.Conv2d(32, 64, 5, 1, 2),
            nn.ReLU(),      #input_size=(16*10*10)
            nn.MaxPool2d(2, 2)  #output_size=(16*5*5)
        )
        self.conv5 = nn.Sequential(
            nn.Conv2d(64, 128, 5, 1, 2),
            nn.ReLU(),      #input_size=(16*10*10)
            nn.MaxPool2d(2, 2)  #output_size=(16*5*5)
        )
        self.conv6 = nn.Sequential(
            nn.Conv2d(128, 256, 5, 1, 2),
            nn.ReLU(),      #input_size=(16*10*10)
            # nn.MaxPool2d(2, 2)  #output_size=(16*5*5)
        )
        self.fc1 = nn.Sequential(
            nn.Linear(256 * 8 * 8, 120),
            nn.ReLU()
        )
        self.fc2 = nn.Sequential(
            nn.Linear(120, 84),
            nn.ReLU()
        )
        self.fc3 = nn.Linear(84, 3)

    def forward(self, x):
        # print(x.size())
        x = self.conv1(x)
        # print(x.size())
        x = self.conv2(x)
        # print(x.size())
        x = self.conv3(x)
        # print(x.size())
        x = self.conv4(x)
        # print(x.size())
        x = self.conv5(x)
        # print(x.size())
        x = self.conv6(x)
        # print(x.size())
        # nn.Linear()
        # print(x.size())
        x = x.view(x.size()[0], -1)
        x = self.fc1(x)
        x = self.fc2(x)
        x = self.fc3(x)
        return x
# net = LeNet12().to(device)
# net = LeNet()
# net = torch.load('F:/yzh/data_process_results/DL/PreprocessDataset/model/Lenet_240.pt') #Load the trained model
# resnet50 = models.resnet18(pretrained=True)
# net = models.squeezenet1_0(num_classes = 5)
# print(net)

#resnet18
# net = models.resnet18(pretrained=False)
# net = models.resnet18(num_classes = 3)
# net.conv1 = nn.Conv2d(1, 64, kernel_size=7, stride=2, padding=3, bias=False)  #Change layer 1 of resnet18 to single channel input
# net.features[0] = nn.Conv2d(1, 96, kernel_size=7, stride=2, padding=3, bias=False)
# for param in resnet50.parameters():

# VGG16

net = models.vgg16(pretrained=False)
# net = net.vgg16(num_classes=3)
net.features[0]=nn.Conv2d(1, 64, kernel_size=7, stride=2, padding=3, bias=False) #Change layer 1 of vgg16 to single channel input
net.classifier[6]=nn.Linear(in_features=4096,out_features=3,bias=True) #Modify the output of the final full connection layer
#     param.requires_grad = False

# fc_inputs = net.fc.in_features

# net.fc = nn.Sequential(

#     nn.Linear(fc_inputs, 256),

#     nn.ReLU(),

#     nn.Dropout(0.4),

#     nn.Linear(256, 5),

#     nn.LogSoftmax(dim=1)

# )

# net.classifier = nn.Sequential(
#     nn.Dropout(p=0.5, inplace=False),

#     nn.Conv2d(512, 5, kernel_size=(1, 1), stride=(1, 1)),

#     nn.ReLU(),

#     nn.LogSoftmax(dim=1)

# )
print(net)
net = net.to(device)

loss_func = nn.CrossEntropyLoss()
# loss_func = nn.NLLLoss()
# optimizer = optim.SGD(net.parameters(), lr=LR, momentum=0.9)
optimizer = optim.Adam(net.parameters(),lr = LR)

def train_and_valid(model, loss_function, optimizer, epochs):

    device = torch.device("cuda" if torch.cuda.is_available() else "cpu")

    history = []

    best_acc = 0.0

    best_epoch = 0

    for epoch in range(epochs):

        epoch_start = time.time()

        print("Epoch: {}/{}".format(epoch+1, epochs))

        model.train()

        train_loss = 0.0

        train_acc = 0.0

        valid_loss = 0.0

        valid_acc = 0.0

        for i, (inputs, labels) in enumerate(trainloader):

            inputs = inputs.to(device)

            labels = labels.to(device)

            optimizer.zero_grad()

            outputs = model(inputs)

            loss = loss_function(outputs, labels)

            loss.backward()

            optimizer.step()

            train_loss += loss.item() * inputs.size(0)

            ret, predictions = torch.max(outputs.data, 1)

            correct_counts = predictions.eq(labels.data.view_as(predictions))

            acc = torch.mean(correct_counts.type(torch.FloatTensor))

            train_acc += acc.item() * inputs.size(0)

        with torch.no_grad():

            model.eval()

            for j, (inputs, labels) in enumerate(testloader):

                inputs = inputs.to(device)

                labels = labels.to(device)

                outputs = model(inputs)

                loss = loss_function(outputs, labels)

                valid_loss += loss.item() * inputs.size(0)

                ret, predictions = torch.max(outputs.data, 1)

                correct_counts = predictions.eq(labels.data.view_as(predictions))

                acc = torch.mean(correct_counts.type(torch.FloatTensor))

                valid_acc += acc.item() * inputs.size(0)

        avg_train_loss = train_loss/train_data_size

        avg_train_acc = train_acc/train_data_size

        avg_valid_loss = valid_loss/valid_data_size

        avg_valid_acc = valid_acc/valid_data_size

        history.append([avg_train_loss, avg_valid_loss, avg_train_acc, avg_valid_acc])

        if best_acc < avg_valid_acc:

            best_acc = avg_valid_acc

            best_epoch = epoch + 1

        epoch_end = time.time()

        print("Epoch: {:03d}, Training: Loss: {:.4f}, Accuracy: {:.4f}%, \n\t\tValidation: Loss: {:.4f}, Accuracy: {:.4f}%, Time: {:.4f}s".format(

            epoch+1, avg_train_loss, avg_train_acc*100, avg_valid_loss, avg_valid_acc*100, epoch_end-epoch_start

        ))

        print("Best Accuracy for validation : {:.4f} at epoch {:03d}".format(best_acc, best_epoch))

        torch.save(model, dataset + savepath + modelname+str(epoch+1)+'.pt')

    return model, history

num_epochs = 40

trained_model, history = train_and_valid(net, loss_func, optimizer, num_epochs)

# torch.save(history, dataset + savepath + modelname+'_history.pt')

history = np.array(history)

plt.plot(history[:, 0:2])

plt.legend(['Tr Loss', 'Val Loss'])

plt.xlabel('Epoch Number')

plt.ylabel('Loss')

plt.ylim(0, 1)

# plt.savefig(dataset + savepath + modelname+'_loss_curve.png')

plt.show()

plt.plot(history[:, 2:4])

plt.legend(['Tr Accuracy', 'Val Accuracy'])

plt.xlabel('Epoch Number')

plt.ylabel('Accuracy')

plt.ylim(0, 1)

# plt.savefig(dataset + savepath + modelname+'_accuracy_curve.png')

plt.show()


# A simple pre-computed KRR example.
# NOTE this does not appear to be a good choice of kernel

# %%
import json
import pandas as pd
import itertools
import numpy as np

from EleMD import EleMD
from tqdm import tqdm
from sklearn.kernel_ridge import KernelRidge

# %%
# Load the data

filename = "data/matbench_expt_gap.json"

with open(filename, 'rb') as f:
    dataframe_data = json.load(f)

df = pd.DataFrame(**dataframe_data)
df = df.sample(frac=1).reset_index(drop=True)

# %%
# Eval train distances

n = 1000

x = df["composition"][:n].values
y = df["gap expt"][:n].values

D = np.zeros((n, n))

mod_petti = EleMD()

iterator = itertools.combinations(range(n), 2)
for i, j in tqdm(iterator):
    D[i, j] = mod_petti.elemd(x[i], x[j])

D = D + D.T

# %%
# Eval test distances

m = 200

x_test = df["composition"][n:n+m].values
y_test = df["gap expt"][n:n+m].values

D_test = np.zeros((m, n))

iterator = itertools.product(range(m), range(n))
for i, j in tqdm(iterator):
    D_test[i, j] = mod_petti.elemd(x[i], x[j])

# %%
# fit KRR
krr = KernelRidge(kernel="precomputed")
lam = 2e-4

def K(D, lam=lam):
    # return np.exp(-D**2/l)
    return lam/(D**2+lam)

krr.fit(K(D), y)

tr_y = krr.predict(K(D))
te_y = krr.predict(K(D_test))

print(f"Train MAE: {np.mean(np.abs(tr_y - y)):.3f}")
print(f"Test MAE: {np.mean(np.abs(te_y - y_test)):.3f}")

print(f"Train RMSE: {np.sqrt(np.mean((tr_y - y)**2)):.3f}")
print(f"Test RMSE: {np.sqrt(np.mean((te_y - y_test)**2)):.3f}")

# %%


# -*- coding: utf-8 -*-
"""
Created on Sat Oct 27 01:43:05 2018

@author: Minkush and Mononito
"""

# K-Means Clustering

# Importing the libraries
import numpy as np
import matplotlib.pyplot as plt
import pandas as pd

# Importing the dataset
dataset = pd.read_excel('TransactionsData.xlsx')
X = dataset.iloc[:,1:8].values

# encoding
from sklearn.preprocessing import OneHotEncoder, LabelEncoder
labelencoder_X = LabelEncoder()
for i in range(7):
    X[:, i] = labelencoder_X.fit_transform(X[:, i])
onehotencoder = OneHotEncoder(categorical_features='all')
X = onehotencoder.fit_transform(X).toarray()
# Feature Scaling
from sklearn.preprocessing import StandardScaler
sc_X = StandardScaler()
X = sc_X.fit_transform(X)

# Using the elbow method to find the optimal number of clusters
from sklearn.cluster import KMeans
wcss = []
for i in range(1, 20):
    kmeans = KMeans(n_clusters = i, init = 'k-means++', random_state = 42)
    kmeans.fit(X)
    wcss.append(kmeans.inertia_)
plt.plot(range(1, 20), wcss)
plt.title('The Elbow Method')
plt.xlabel('Number of clusters')
plt.ylabel('WCSS')
plt.show()

# Fitting K-Means to the dataset
kmeans = KMeans(n_clusters = 30, init = 'k-means++', random_state = 42)
kmeans.fit(X)


#!/usr/bin/env python
"""
coding=utf-8

Main entry point into Moby-Dick project.

There aren't great batteries included examples for modeling text with deep learning, so I've built out this repo to
contain starter code for:

 - Text processing: Processing text to be utilized with keras (text pre-processing, converting to indices,
 padding) - Pre-trained embedding: Using a pre-trained text embedding (GoogleNews 300) with keras (translating words
 to a point in \mathbb{R}^{300}) - Convolutional architecture: Modeling text with a convolutional architecture (
 functionally similar to Ngrams) - RNN architecture: Modeling text with a Recurrent Neural Net (RNN) architecture (
 functionally similar to a rolling window)

"""
import glob
import logging
import ntpath
import os

from collections import defaultdict

import numpy
import pandas

from gensim.utils import simple_preprocess

import lib
import resources
import models

def main():
    """
    Main function documentation template
    :return: None
    :rtype: None
    """
    logging.getLogger().setLevel(level=logging.DEBUG)

    # Extract data from upstream
    articles, embedding_matrix, word_to_index = extract()

    # Transform data for modeling
    articles, embedding_matrix, word_to_index = transform(articles, embedding_matrix, word_to_index)

    # Model data / predict on data
    model(articles, embedding_matrix, word_to_index)

    # TODO Load. What is using this down stream?
    load()
    pass

def extract():
    """
    Extract necessary data / resources from upstream. This method will:

     - Validate that newsgroup data set is available, and read in
     - Validate that text embeddings are available, and read in
     - Validate that text to embedding index lookup is available, and read in

    :return: observations, embedding_matrix, word_to_index
    :rtype: (pandas.DataFrame, numpy.array, dict)
    """

    logging.info('Begin extract')
    logging.info('Performing extract for batch: {}, from newgroup_path: {}'
                 .format(lib.get_batch_name(), lib.get_conf('newsgroup_path')))

    # Download resources

    # Confirm newsgroup data set is downloaded
    resources.download_newsgroup()

    # Confirm that embedding is downloaded
    resources.download_embedding()

    # Extract resources from file system

    # Newsgroup20: Get list of all candidate documents
    glob_pattern = os.path.join(lib.get_conf('newsgroup_path'), '*', '*')
    logging.info('Searching for glob_pattern: {}'.format(glob_pattern))
    document_candidates = glob.glob(glob_pattern)

    # Newsgroup20: Create observations data set
    observations = pandas.DataFrame(document_candidates, columns=['document_path'])
    logging.info('Shape of observations data frame created from glob matches: {}'.format(observations.shape))

    # Newsgroup20: Re-order rows
    observations = observations.sample(frac=1)

    # Newsgroup20: Subset number of observations, if it's a test run
    if lib.get_conf('test_run'):
        logging.info('Reducing file size for test run')
        observations = observations.head(100)
        logging.info('Test run number of records: {}'.format(len(observations.index)))

    # Embedding: Load embedding
    embedding_matrix, word_to_index = resources.create_embedding_matrix()
    logging.info('word_to_index max index: {}'.format(max(word_to_index.values())))

    # Archive schema and return
    lib.archive_dataset_schemas('extract', locals(), globals())
    logging.info('End extract')
    return observations, embedding_matrix, word_to_index

def transform(observations, embedding_matrix, word_to_index):
    """
    Transform data and resources to be ready for model consumption. This includes:

     - Pulling document category
     - Pulling raw text (stripped of headers)
     - Normalizing text, converting text to an array of tokens, and converting array of tokens to array of embedding
     indices
     - Padding array of indices, so that all arrays of indices are the same length

    :param observations: A Pandas DataFrame containing the field padded_indices
    :type observations: pandas.DataFrame
    :param embedding_matrix: An embedding matrix, with shape (n,m), where n is the number of words, and m is the
    dimensionality of the embedding
    :type embedding_matrix: numpy.array
    :param word_to_index: A mapping from words (strings), to their index in the embedding matrix. For example
    embedding_matrix[word_to_index['pineapple']] would give the embedding vector for the word 'pineapple'
    :type: {str:int}
    :return: articles, embedding_matrix, word_to_index
    :rtype: (pandas.DataFrame, numpy.array, dict)
    """

    logging.info('Begin transform')

    # Transform embedding resources
    # Embedding: Update embedding to map any unknown words (words not in training vocabulary) to the unknown value
    default_dict_instance = defaultdict(lambda: word_to_index['UNK'])
    default_dict_instance.update(word_to_index)
    word_to_index = default_dict_instance

    # Transform newsgroup20 data set
    # Newsgroup20: Extract article filename from document path
    observations['filename'] = observations['document_path'].apply(lambda x: ntpath.basename(x))

    # Newsgroup20: Extract article category from document path
    observations['category'] = observations['document_path'].apply(lambda x: ntpath.basename(os.path.dirname(x)))

    # Newsgroup20: Extract article text (and strip article headers), from document path
    observations['text'] = observations['document_path'].apply(lambda x: lib.strip_header(open(x).readlines()))

    # Newsgroup20: Convert text to normalized tokens. Unknown tokens will map to 'UNK'.
    observations['tokens'] = observations['text'].apply(simple_preprocess)

    # Newsgroup20: Convert tokens to indices
    observations['indices'] = observations['tokens'].apply(lambda token_list: map(lambda token: word_to_index[token],
                                                                                token_list))
    observations['indices'] = observations['indices'].apply(lambda x: numpy.array(x))

    # Newsgroup20: Pad indices list with zeros, so that every article's list of indices is the same length
    observations['padded_indices'] = observations['indices'].apply(lib.pad_sequence)

    # Newsgroup20: Create label one hot encoder, and encode labels
    label_encoder = lib.create_label_encoder(set(observations['category']))
    observations['category_encoded'] = observations['category'].apply(lambda x: label_encoder[x])

    # Set up modeling input
    observations['modeling_input'] = observations['padded_indices']

    # Archive schema and return
    lib.archive_dataset_schemas('transform', locals(), globals())
    logging.info('End transform')
    return observations, embedding_matrix, word_to_index

def model(observations, embedding_matrix, word_to_index):
    """
    Train models, as appropriate.

    The Pandas DataFrame observations should contain the fields `modeling_input` (where each member is an iterable of
    the same length), and `category_encoded` (where each member is an iterable of the same length)

    :param observations: A Pandas DataFrame containing the field padded_indices
    :type observations: pandas.DataFrame
    :param embedding_matrix: An embedding matrix, with shape (n,m), where n is the number of words, and m is the
    dimensionality of the embedding
    :type embedding_matrix: numpy.array
    :param word_to_index: A mapping from words (strings), to their index in the embedding matrix. For example
    embedding_matrix[word_to_index['pineapple']] would give the embedding vector for the word 'pineapple'
    :type: {str:int}
    :return: None
    :rtype: None
    """

    logging.info('Begin model')

    # Reference variables

    # Create train and test data sets

    train_test_mask = numpy.random.random(size=len(observations.index))
    num_train = sum(train_test_mask < .8)
    num_validate = sum(train_test_mask >= .8)
    logging.info('Proceeding w/ {} train observations, and {} test observations'.format(num_train, num_validate))

    x_train = observations['modeling_input'][train_test_mask < .8].tolist()
    y_train = observations['category_encoded'][train_test_mask < .8].tolist()
    x_test = observations['modeling_input'][train_test_mask >= .8].tolist()
    y_test = observations['category_encoded'][train_test_mask >= .8].tolist()

    # Convert x and y vectors to numpy objects
    x_train = numpy.array(x_train, dtype=object)
    y_train = numpy.array(y_train)
    x_test = numpy.array(x_test, dtype=object)
    y_test = numpy.array(y_test)

    logging.info('x_train shape: {}, y_train shape: {}, '
                 'x_test shape: {}, y_test shape: {}'.format(x_train.shape, y_train.shape, x_test.shape, y_test.shape))

    # If required, train model
    if lib.get_conf('train_model'):
        logging.info('Creating and training model')

        classification_model = models.gen_conv_model(observations, embedding_matrix, word_to_index)

        # Train model
        classification_model.fit(x_train, y_train, batch_size=128, epochs=20, validation_data=(x_test, y_test))

        logging.info('Finished creating and training model')

    # Validate model

    # Archive schema and return
    lib.archive_dataset_schemas('transform', locals(), globals())
    logging.info('End model')
    pass

def load():
    pass

# Main section
if __name__ == '__main__':
    main()


from bs4 import BeautifulSoup
import requests
from selenium import webdriver
from selenium.webdriver.common.keys import Keys
import time
import os
import json

def remove_spaces(spaces):
    spaces = spaces.strip().replace(" +","")
    spaces = " ".join(spaces.split())
    return spaces

PATH = os.getcwd()
JSON_FILEPATH = os.path.join(PATH,'linkedin_job_finder.json')
if os.path.exists(JSON_FILEPATH) :
    flag = 0
    with open(JSON_FILEPATH, 'r') as f:
        data_master = json.load(f)
else:
    flag = 1
    with open(JSON_FILEPATH, 'w') as fp:
        pass

options = webdriver.ChromeOptions()
options.add_argument('--disable-gpu')
options.add_argument('--no-sandbox')
options.add_argument('--headless')
driver = webdriver.Chrome(options = options)

data = []

url ="https://www.linkedin.com/jobs/search/?currentJobId=2599978326&jobPostingId=2599472731&pivotType=similarJobs"
driver.get(url)
content = driver.page_source
soup = BeautifulSoup(content, "html.parser")

st_divs1 = soup.find_all('div',{"class" : "base-card base-card--link base-search-card base-search-card--link job-search-card"})
print(len(st_divs1))

for divs in st_divs1 :

    print("*"*20)

    JOBID = ""
    TITLE = ""
    COMPANY = ""
    PLACE = ""
    DATE = ""
    LINK = ""
    DESCRIPTION = ""
    JOB_FUNCTION = ""
    EMPLOYMENT_TYPE = ""
    INDUSTRIES = ""

    job_id = divs['data-entity-urn'].split(':')[-1]
    job_id = remove_spaces(job_id)
    print(job_id)
    JOBID = job_id

    links = divs.find('a',{"class" :"base-card__full-link"})['href']
    links = remove_spaces(links)
    print(links)
    LINK = links

    title = divs.find('h3',{"class" : "base-search-card__title"}).text
    title = remove_spaces(title)
    print(title)
    TITLE = title

    company = divs.find('h4',{"class" : "base-search-card__subtitle"}).text
    company = remove_spaces(company)
    print(company)
    COMPANY = company

    location = divs.find('span',{"class" : "job-search-card__location"}).text
    location = remove_spaces(location)
    print(location)
    PLACE = location

    try :
        date_time = divs.find('time',{"class" : "job-search-card__listdate--new"})['datetime']
        date_time = remove_spaces(date_time)
        print(date_time)
        DATE = date_time

    except Exception as e :
        print(e)
        pass

    driver.get(links)
    content = driver.page_source
    soup_1 = BeautifulSoup(content, "html.parser")
    time.sleep(3)

    try :
        description = soup_1.find('div',{"class" : "show-more-less-html__markup"}).text
        description = remove_spaces(description)
        print(description)
        DESCRIPTION = description

    except Exception as e :
        print(e)
        pass

    try :
        info_div = soup_1.find_all('li',{"class" : "description__job-criteria-item"})
        for info in info_div :
            employment_type = remove_spaces(info.find('h3').text)
            employment_desc = remove_spaces(info.find('span').text)
            print(employment_type,employment_desc)
            if 'employment' in employment_type.lower() :
                EMPLOYMENT_TYPE = employment_desc
            elif 'job' in employment_type.lower() :
                JOB_FUNCTION = employment_desc
            elif 'industries' in employment_type.lower() :
                INDUSTRIES = employment_desc

    except Exception as e:
        print(e)
        pass

    scraper_data = {}
    scraper_data['job_id'] = JOBID
    scraper_data['title'] = TITLE
    scraper_data['company'] = COMPANY
    scraper_data['place'] = PLACE
    scraper_data['date'] = DATE
    scraper_data['link'] = LINK
    scraper_data['description'] = DESCRIPTION
    scraper_data['job_function'] = JOB_FUNCTION
    scraper_data['employment_type'] = EMPLOYMENT_TYPE
    scraper_data['industries'] = INDUSTRIES
    data.append(scraper_data)

if data == [] :
    raise Exception('The data object is empty . Please run the Python File once again ')

if flag == 0 :
    for x in data :
        if x not in data_master:
            data_master.append(x)

    seen = []
    for x in data_master:
        if x not in seen:
            seen.append(x)

    with open(JSON_FILEPATH,'w') as file :
        json.dump(seen,file,ensure_ascii=False, indent=1)

elif flag == 1 :
    with open(JSON_FILEPATH,'w') as file :
        json.dump(data,file,ensure_ascii=False, indent=1)

driver.quit()


from flask import Blueprint, render_template, request, redirect, session, jsonify, flash, url_for, abort
from functools import wraps
import hashlib
import json
from datetime import datetime

#custom imports
from .models import Users, Shortlist
from .models import Jobs
shorlist_obj=Shortlist()
user_object = Users()
jobs=Jobs()
admin = Blueprint("admin", __name__, template_folder='templates', static_folder='static/admin',
                   static_url_path='static/admin')

@admin.errorhandler(404)
def page_not_found(error):
    return render_template('admin/404.html'), 404

@admin.route('/add_job',methods=['POST','GET'])
def add_job():
    try:
        if request.method == 'POST':
            job={}
            job['category']=request.form.get('category')
            job['title']=request.form.get('description')
            job['description']=request.form.get('description')
            job['skills']= request.form.get('skills').split(',')
            job['aptitude']=[]
            job['personality']=[]
            job['created_on']= datetime.now()
            result=jobs.put_job(job)
            return redirect(url_for('admin.add_aptitude',job_id=result))
        else:
            return render_template('admin/index.html')
    except Exception as error:
        print(error)
        return render_template('admin/index.html')

@admin.route('/add_aptitude/<job_id>',methods=['POST','GET'])
def add_aptitude(job_id):
    print(job_id)
    try:
        job={}
        job['job_id']=job_id
        job['is_apti']=True
        if request.method == 'POST':
            apti=[]
            for i in range(10):
                tmp={}
                tmp['question']=request.form.get('question'+str(i+1))
                tmp['options']=request.form.get('options'+str(i+1)).split(',')
                tmp['answer']=request.form.get('answer'+str(i+1))
                apti.append(tmp)
            print(apti)
            jobs.put_aptitude(job_id,apti)
            return redirect(url_for('admin.add_personality',job_id=job_id))
        else:
            return render_template('admin/test.html',job=job)

    except Exception as error:
        job={}
        job['job_id']=job_id
        job['is_apti']=False
        print('In exception: ',error)
        return render_template('admin/test.html',job=job)

@admin.route('/dashboard')
def dashboard():
    alljobs=jobs.get_all_jobs()
    print('\n\nall jobs: ',alljobs)
    return render_template('admin/dashboard.html',alljobs=alljobs)

@admin.route('/shortlist')
def shortlist():
    print(request.args.get('job_id'))
    profiles=shorlist_obj.get_profiles(request.args.get('job_id'))
    print('\n\nall jobs: ',profiles)
    return render_template('admin/shortlist.html',profiles=profiles)

@admin.route('/delete/<job_id>',methods=['POST','GET'])
def delete(job_id):
    try:
        print("Im here"*100,job_id)
        jobs.delete_job(job_id)

        return redirect(url_for('admin.dashboard'))
    except Exception as error:

        return redirect(url_for('admin.dashboard'))

@admin.route('/add_personality/<job_id>',methods=['POST','GET'])
def add_personality(job_id):
    print(job_id)
    try:
        job={}
        job['job_id']=job_id
        job['is_apti']=False
        if request.method == 'POST':
            apti=[]
            for i in range(10):
                tmp={}
                tmp['question']=request.form.get('question'+str(i+1))
                tmp['options']=request.form.get('options'+str(i+1)).split(',')
                tmp['answer']=request.form.get('answer'+str(i+1))
                apti.append(tmp)
            print(apti)
            jobs.put_personality(job_id,apti)
            return redirect(url_for('admin.dashboard'))
        else:
            return render_template('admin/test.html',job_id=job_id)

    except Exception as error:
        print('In exception: ',error)

        job={}
        job['job_id']=job_id
        job['is_apti']=False

        return render_template('admin/test.html',job=job)


# from IPython import get_ipython
# get_ipython().magic('reset -sf')
import numpy as np
import cvxpy as cv
import matplotlib as mp
# mp.use('QT4Agg')
import matplotlib.pyplot as pl
import pylab as py
import os, sys, time, copy#Λ
# haha=np.array([[0,1,2],
#                [3,4,5],
#                [6,7,8]])
print('\n'*100)
tt = time.time()
# print('\ntime elapsed=', time.time()-tt)
# exec(open('ls_perm_meas_data.py').read())
# import ls_perm_meas_data as ct
np.set_printoptions(precision=3)
# PTl[np.asmatrix([1,3,5]).T, [1,3,5]]
# legend(loc='upper center', bbox_to_anchor=(0.5, 1.05))

nn = 16  # 2**4

vo = np.asmatrix([0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1])
v1 = np.asmatrix([0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1])
v2 = np.asmatrix([0, 0, 0, 0, 1, 1, 1, 1, 0, 0, 0, 0, 1, 1, 1, 1])
v3 = np.asmatrix([0, 0, 1, 1, 0, 0, 1, 1, 0, 0, 1, 1, 0, 0, 1, 1])
v4 = np.asmatrix([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0])
v51 = np.asmatrix([0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0])
v52 = np.asmatrix([0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0])
vu = np.asmatrix([1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1])
pp = cv.Variable(nn, 1)

obj = [cv.Minimize(vo * pp),
       cv.Maximize(vo * pp)]
tag = ['min p4', 'max p4']
for idx in np.arange(2):
    cst = [  # pp <= 1,
           0 <= pp,
           vu * pp == 1,
           v1 * pp == .9,
           v2 * pp == .9,
           v3 * pp == .1,
           v4 * pp == .7 * v3 * pp,
           v52 * pp == .6 * v51 * pp]
    prb = cv.Problem(obj[idx], cst)
    prb.solve()
    print(prb.status)
    print(tag[idx], prb.value)
    # print(pp.value)


# -*- coding: utf-8 -*-

"""
@date: 2020/8/29 上午11:05
@file: base_dataset.py
@author: zj
@description:
"""

import os
import torch
import numpy as np
from PIL import Image
from torch.utils.data import Dataset

from .clipsample import SegmentedSample, DenseSample

class VideoRecord(object):
    def __init__(self, row):
        self._data = row

    @property
    def path(self):
        return self._data[0]

    @property
    def num_frames(self):
        return int(self._data[1])

    @property
    def label(self):
        return int(self._data[2])

class BaseDataset(Dataset):

    def __init__(self,
                 data_dir,
                 annotation_dir,
                 modality="RGB",
                 sample_strategy='SegSample',
                 clip_len=1,
                 frame_interval=1,
                 num_clips=3,
                 is_train=True,
                 transform=None):
        assert isinstance(modality, str) and modality in ('RGB', 'RGBDiff')
        assert isinstance(sample_strategy, str) and sample_strategy in ('SegSample', 'DenseSample')

        if modality == 'RGB':
            assert clip_len == 1
        elif modality == 'RGBDiff':
            assert clip_len == 5
            # Diff needs one more image to calculate diff
            clip_len += 1
        else:
            raise ValueError(f'{self.modality} does not exist')

        self.data_dir = data_dir
        self.annotation_dir = annotation_dir
        self.modality = modality
        self.sample_strategy = sample_strategy
        self.clip_len = clip_len
        self.frame_interval = frame_interval
        self.num_clips = num_clips
        self.is_train = is_train
        self.transform = transform

        self.video_list = None
        self.cate_list = None
        self.img_num_list = None
        self.sampler = None
        # RawFrames下标从0开始，比如UCF101/HMDB51，也有用1开始，比如JESTER
        self.start_index = 0
        # RawFrames图像命令前缀，比如UCF101/HMDB51使用img_，JESTER没有
        self.img_prefix = 'img_'

    def _update_video(self, annotation_dir, is_train=True):
        pass

    def _update_class(self):
        pass

    def _sample_frames(self):
        if self.sample_strategy == 'SegSample':
            self.clip_sample = SegmentedSample(self.clip_len,
                                           self.frame_interval,
                                           self.num_clips,
                                           is_train=self.is_train,
                                           start_index=self.start_index)
        elif self.sample_strategy == 'DenseSample':
            self.clip_sample = DenseSample(self.clip_len,
                                       self.frame_interval,
                                       self.num_clips,
                                       is_train=self.is_train,
                                       start_index=self.start_index)
        else:
            raise ValueError(f'{self.sample_strategy} does not exist')

    def __getitem__(self, index: int):
        """
        从选定的视频文件夹中随机选取T帧，则返回(T, C, H, W)，其中T表示num_segs
        """
        assert index < len(self.video_list)
        record = self.video_list[index]
        target = record.label

        clip_offsets = self.clip_sample(record.num_frames)

        video_path = os.path.join(self.data_dir, record.path)
        image_list = list()
        for offset in clip_offsets:
            if 'RGB' == self.modality:
                img_path = os.path.join(video_path, '{}{:0>5d}.jpg'.format(self.img_prefix, offset))
                img = np.array(Image.open(img_path))

                if self.transform:
                    img = self.transform(img)
                image_list.append(img)
            if 'RGBDiff' == self.modality:
                tmp_list = list()
                for clip in range(self.clip_len):
                    img_path = os.path.join(video_path, '{}{:0>5d}.jpg'.format(self.img_prefix, offset + clip))
                    img = np.array(Image.open(img_path))

                    tmp_list.append(img)
                for clip in reversed(range(1, self.clip_len)):
                    img = tmp_list[clip] - tmp_list[clip - 1]
                    if self.transform:
                        img = self.transform(img)
                    image_list.append(img)
        # [T, C, H, W] -> [C, T, H, W]
        image = torch.stack(image_list).transpose(0, 1)

        return image, target

    def __len__(self) -> int:
        return len(self.video_list)


# coding=UTF-8
#!flask/bin/python
from flask import Flask, jsonify, request, abort, url_for, send_from_directory
from produtos import RepositorioProdutos, ElementoInexistente, ElementoInvalido
from operacoes import RepositorioOperacoes
import os

HTTP_200_OK = 200
HTTP_201_CREATED = 201
HTTP_400_BAD_REQUEST = 400
HTTP_404_NOT_FOUND  = 404

app = Flask(__name__, static_folder='static', static_url_path='/ui')
repProdutos = RepositorioProdutos()

@app.route("/")
def hello():
    return send_from_directory(app.static_folder, 'index.html')

@app.route('/app/produtos', methods=['GET'])
def list_Produtos():
    produtos = repProdutos.listar()
    return jsonify({'produtos': produtos})

@app.route('/app/produtos', methods=['POST'])
def create_prod():
    try:
        if not request.json['id']:
            produto = repProdutos.adicionar(request.json)
        else:
            produto = repProdutos.atualizar(request.json)

        return jsonify({'resultado': produto}), HTTP_201_CREATED
    except ElementoInvalido:
       abort(HTTP_400_BAD_REQUEST)

if __name__ == "__main__":
    app.run(host=os.getenv('IP', '0.0.0.0'),port=int(os.getenv('PORT', 8080)))


import argparse
import itertools
import pandas as pd
import statsmodels.api as sm

import hnelib.runner
import hnelib.pandas

import usfhn.datasets
import usfhn.constants
import usfhn.measurements
import usfhn.views
import usfhn.non_us
import usfhn.fieldwork
import usfhn.attrition
import usfhn.careers
import usfhn.interdisciplinarity
import usfhn.new_hires
import usfhn.self_hiring
import usfhn.taxonomy
import usfhn.rank
import usfhn.gender
import usfhn.pool_reduction
import usfhn.demographics
import usfhn.closedness
import usfhn.webs
import usfhn.non_doctorate
import usfhn.stats_utils

################################################################################
#
#
# Utilities for standard names
#
#
################################################################################
VARIABLE_TO_COLUMN = {
    'ginis': 'GiniCoefficient',
    'gender': 'FractionFemale',
    'self-hires': 'SelfHiresFraction',
    'self-hire': 'SelfHire',
    'non-us': 'NonUS',
    'faculty-rank': 'Rank',
    'seniority': 'Senior',
    'new-hire': 'NewHire',
    'year': 'Year',
    'career-age': 'CareerAge',
}

# higher priority = happens earlier in the path
FLAG_PARAMETER_PRIORITY = {
    'by_year': 0,
}

def is_flag_parameter(parameter):
    """
    so I get in something like `by_faculty_rank`

    return false if doesn't start with `by_`

    replace '_' with '-'

    return true if is in VARIABLE_TO_COLUMN
    """
    if not parameter.startswith('by_'):
        return False

    parameter = parameter.replace('by_', '')
    parameter = parameter.replace('_', '-')

    return parameter in VARIABLE_TO_COLUMN

def get_flag_parameters_from_kwargs(**kwargs):
    """
    we're going to return the flag parameters in a specific order because
    sometimes there are multiple qualifiers
    """
    parameters = [k for k, v in kwargs.items() if v and is_flag_parameter(k)]

    max_priority = max(FLAG_PARAMETER_PRIORITY.values()) + 1
    return sorted(parameters, key=lambda p: FLAG_PARAMETER_PRIORITY.get(p, max_priority), reverse=True)

def get_variable_from_flag_parameter(string):
    string = string.replace('by_', '')
    string = string.replace('_', '-')
    return string

def is_variable_qualifier(string):
    if not string.startswith('by-'):
        return False

    string = string.replace('by-', '')

    return string in VARIABLE_TO_COLUMN

def get_groupby_cols_from_kwargs(exclude=[], **kwargs):
    parameters = get_flag_parameters_from_kwargs(**kwargs)
    variables = [get_variable_from_flag_parameter(p) for p in parameters]
    groupby_cols = [VARIABLE_TO_COLUMN[v] for v in variables]
    groupby_cols = [c for c in groupby_cols if c not in exclude]
    return groupby_cols

def get_variable_dataframe(variable, suffix=None, item_kwargs={}, **kwargs):
    flag_parameters = get_flag_parameters_from_kwargs(**kwargs)
    qualifiers = [p.replace('_', '-') for p in flag_parameters]

    parts = [variable] + qualifiers + [suffix]
    path = "/".join([p for p in parts if p])
    df = runner.get(path, **item_kwargs)

    return df

def add_groupby_annotations_to_df(
    df,
    by_gender=False,
    by_new_hire=False,
    by_non_us=False,
    by_faculty_rank=False,
    by_seniority=False,
    by_career_age=False,
    by_self_hire=False,
    explode_gender=False,
    **kwargs,
):
    cols = list(df.columns)

    groupby_cols = []

    if by_gender:
        df = usfhn.gender.annotate_gender(df, explode_gender=explode_gender)
        groupby_cols.append('Gender')

    if by_career_age:
        df = usfhn.careers.annotate_career_age(df)
        groupby_cols.append('CareerAge')

    if by_faculty_rank:
        df = usfhn.rank.annotate_faculty_rank(df)
        groupby_cols.append('Rank')

    if by_seniority:
        df = usfhn.rank.annotate_seniority(df)
        groupby_cols.append('Senior')

    if by_non_us:
        df = usfhn.non_us.annotate_non_us(df)
        groupby_cols.append('NonUS')

    if by_new_hire:
        df = usfhn.new_hires.annotate_new_hires(df)
        groupby_cols.append('NewHire')

    if by_self_hire:
        df = usfhn.self_hiring.annotate_self_hires(df)
        groupby_cols.append('SelfHire')

    df = df[
        cols + groupby_cols
    ].drop_duplicates()

    return df, groupby_cols

################################################################################
#
#
# Expanders and stats collections
#
#
################################################################################
RANKS_EXPANDER = {'rank_type': ['production', 'prestige']}
ALL_RANKS_EXPANDER = {'rank_type': ['production', 'doctoral-institution', 'employing-institution']}

def get_attrition_risk_stats_functions():
    return {
        'taxonomy': usfhn.attrition.get_taxonomy_attrition_risk,
        'gender': usfhn.attrition.get_gender_attrition_risk,
        'self-hires': usfhn.attrition.get_self_hire_attrition_risk,
        'institution': {
            'do': usfhn.attrition.get_institution_attrition_risk,
            'prefix_expansions': {'institution_column': ['DegreeInstitutionId', 'InstitutionId']},
        },
        'us': usfhn.attrition.get_non_us_attrition_risk,
        'non-us-by-is-english': usfhn.attrition.get_non_us_by_is_english_attrition_risk,
        'rank-change': {
            'do': usfhn.attrition.get_rank_change_attrition_risk,
            'directory_expansions': RANKS_EXPANDER,
        },
    }

def get_rank_stats_functions():
    return {
        'directory_expansions': RANKS_EXPANDER,
        'df': usfhn.rank.get_ranks_df_for_stats,
        'change': usfhn.rank.get_rank_change_df,
        'hierarchy-stats': usfhn.rank.get_hierarchy_stats,
        'mean-change': usfhn.rank.get_mean_rank_change,
        'movement-distance': usfhn.rank.get_gendered_movement_distance_significance,
        'gender': {
            'kwargs': {'by_gender': True},
            'hierarchy-stats': usfhn.rank.get_hierarchy_stats,
            'mean-change': usfhn.rank.get_mean_rank_change,
            'placements': usfhn.rank.get_placements,
        },
        'seniority': {
            'kwargs': {'by_seniority': True},
            'hierarchy-stats': usfhn.rank.get_hierarchy_stats,
            'mean-change': usfhn.rank.get_mean_rank_change,
        },
    }

################################################################################
#
#
# basics
#
#
################################################################################
def get_faculty_hiring_network(by_year=False, use_closedness_data=False, by_gender=False):
    """
    Columns:
    - DegreeInstitutionId: producing institution id
    - InstitutionId: employing institution id
    - Count: faculty at InstitutionId with PhDs from DegreeInstitutionId
    - OutDegree: total faculty with PhDs from DegreeInstitutionId
    - InDegree: total faculty employed by InstitutionId
    - TotalCount: total faculty

    we include zeroes in when the level is academia because we know that these
    are phd-granting institutions, so they are producers.

    we do not know whether a particular phd-granting institution is phd granting
    in any particular field/area/umbrella, however, so we exclude them to arrive
    at a lower bound.
    """
    groupby_cols = [
        'TaxonomyLevel',
        'TaxonomyValue',
    ]

    df_name = 'closedness_data' if use_closedness_data else 'data'

    if by_year:
        groupby_cols.append('Year')

    df = usfhn.datasets.get_dataset_df(df_name, by_year=by_year)

    df = usfhn.fieldwork.linearize_df_taxonomy_hierarchy(df)

    df = df[
        groupby_cols + [
            'InstitutionId',
            'DegreeInstitutionId',
            'PersonId',
        ]
    ].drop_duplicates()

    df, extra_groupby_cols = add_groupby_annotations_to_df(df, by_gender=by_gender)
    groupby_cols += extra_groupby_cols

    df['Count'] = df.groupby(
        ['InstitutionId', 'DegreeInstitutionId'] + groupby_cols
    )['PersonId'].transform('nunique').fillna(0)

    df['OutDegree'] = df.groupby(
        ['DegreeInstitutionId'] + groupby_cols
    )['PersonId'].transform('nunique').fillna(0)

    df['InDegree'] = df.groupby(
        ['InstitutionId'] + groupby_cols
    )['PersonId'].transform('nunique').fillna(0)

    df['TotalCount'] = df.groupby(
        groupby_cols
    )['PersonId'].transform('nunique').fillna(0)

    df = df.drop(columns=['PersonId']).drop_duplicates()

    df = df[
        (df['Count'] > 0)
        |
        (df['TaxonomyLevel'] == 'Academia')
    ]

    return df

def get_faculty_production(by_year=False, new_hires=False, non_new_hires=False, non_us=False):
    """
    We include zeroes in when the level is academia because we KNOW that these
    are PhD-granting institutions, so they are Producers.

    We do not know whether a particular PhD-granting institution is PhD granting
    in any particular field/area/umbrella, however, so we exclude them to arrive
    at a lower bound.
    """
    groupby_cols = [
        'TaxonomyLevel',
        'TaxonomyValue',
    ]

    dataset_name = 'closedness_data' if non_us else 'data'

    if by_year:
        groupby_cols.append('Year')

    df = usfhn.datasets.get_dataset_df(dataset_name, by_year=by_year)
    df = usfhn.fieldwork.linearize_df_taxonomy_hierarchy(df)

    df = df[
        groupby_cols + [
            'PersonId',
            'DegreeInstitutionId',
            'InstitutionId',
        ]
    ].drop_duplicates()

    if new_hires or non_new_hires:
        if by_year:
            new_hires_df = runner.get('new-hire/by-year/df')
        else:
            new_hires_df = runner.get('new-hire/df')

        new_hires_df['NewHire'] = True

        df = df.merge(
            new_hires_df,
            on=list(df.columns),
            how='left',
        )

        df['NewHire'] = df['NewHire'].fillna(False)

        if non_new_hires:
            df = df[~df['NewHire']]
        else:
            df = df[df['NewHire']]

        df = df.drop(columns=['NewHire'])

    df['ProductionCount'] = df.groupby(
        groupby_cols + ['DegreeInstitutionId']
    )['PersonId'].transform('nunique').fillna(0)

    df = df[
        (df['ProductionCount'] > 0)
        |
        (df['TaxonomyLevel'] == 'Academia')
    ]

    df['TotalProduction'] = df.groupby(groupby_cols)['PersonId'].transform('nunique')

    df = df.drop(columns=['PersonId']).drop_duplicates()

    df['ProductionFraction'] = df['ProductionCount'] / df['TotalProduction']
    df['ProductionPercent'] = 100 * df['ProductionFraction']

    return df

def get_faculty_hiring(by_year=False):
    """
    We include zeroes in when the level is academia because we KNOW that these
    are PhD-granting institutions, so they are Producers.

    We do not know whether a particular PhD-granting institution is PhD granting
    in any particular field/area/umbrella, however, so we exclude them to arrive
    at a lower bound.
    """
    groupby_cols = [
        'TaxonomyLevel',
        'TaxonomyValue',
    ]

    if by_year:
        groupby_cols.append('Year')

    df = usfhn.datasets.get_dataset_df('data', by_year=by_year)

    df = usfhn.fieldwork.linearize_df_taxonomy_hierarchy(df)

    df = df[
        groupby_cols + [
            'DegreeInstitutionId',
            'InstitutionId',
            'PersonId',
        ]
    ].drop_duplicates()

    df['HiredFaculty'] = df.groupby(
        groupby_cols + ['InstitutionId', 'DegreeInstitutionId',]
    )['PersonId'].transform('nunique').fillna(0)

    df['TotalHiredFaculty'] = df.groupby(groupby_cols)['PersonId'].transform('nunique')

    df = df.drop(columns=['PersonId']).drop_duplicates()

    df = df[
        (df['HiredFaculty'] > 0)
        |
        (df['TaxonomyLevel'] == 'Academia')
    ]

    df['HiredFacultyFraction'] = df['HiredFaculty'] / df['TotalHiredFaculty']

    return df

def get_lorenz_curves(by_year=False, new_hires=False, non_new_hires=False):
    groupby_cols = [
        'TaxonomyLevel',
        'TaxonomyValue',
    ]

    if by_year:
        groupby_cols.append('Year')

        if new_hires:
            df = runner.get('new-hire/by-year/production')
        else:
            df = runner.get('basics/by-year/production')
    else:
        if new_hires:
            df = runner.get('new-hire/production')
        elif non_new_hires:
            df = runner.get('non-new-hire/production')
        else:
            df = runner.get('basics/production')

    curves = []
    for _, rows in df.groupby(groupby_cols):
        rows = rows.drop_duplicates(subset=['DegreeInstitutionId'])
        curve = usfhn.measurements.lorenz_curve(rows['ProductionFraction'])
        for col in groupby_cols:
            curve[col] = rows.iloc[0][col]

        curves.append(curve)

    return pd.concat(curves)

################################################################################
#
#
# Changes over time
#
#
################################################################################
def run_ols_and_get_slopes(variable, p_threshold=.05, **kwargs):
    df = get_variable_dataframe(variable, suffix='df', by_year=True, **kwargs)

    groupby_cols = ['TaxonomyLevel', 'TaxonomyValue']
    groupby_cols += get_groupby_cols_from_kwargs(**kwargs)

    value_col = VARIABLE_TO_COLUMN[variable]

    result_rows = []
    for _, rows in df.groupby(groupby_cols):
        groupby_info = {c: rows.iloc[0][c] for c in groupby_cols}

        X = sm.add_constant(rows['Year'] - (min(rows['Year']) - 1))
        Y = rows[value_col]
        model = sm.OLS(Y, X)
        result = model.fit()

        result_rows.append({
            **groupby_info,
            'Slope': result.params['Year'],
            'R^2': result.rsquared,
            'P': result.pvalues['Year'],
            'Significant': p_threshold > result.pvalues['Year'],
        })

    df = pd.DataFrame(result_rows)

    df = usfhn.stats_utils.correct_multiple_hypotheses(df)
    df['Significant'] = df['PCorrected'] < p_threshold

    return pd.DataFrame(result_rows)

def degree_years_by_taxonomy():
    df = usfhn.datasets.CURRENT_DATASET.closedness_data[
        [
            'PersonId',
            'DegreeYear',
            'Taxonomy',
        ]
    ].drop_duplicates()

    df = usfhn.fieldwork.linearize_df_taxonomy_hierarchy(df)

    return df

def get_years_until_like_new_hires(variable, significant=True):
    value_col = VARIABLE_TO_COLUMN[variable]

    df = get_variable_dataframe(variable, suffix='df', by_new_hire=True)
    df = hnelib.pandas.aggregate_df_over_column(
        df,
        agg_col='NewHire',
        join_cols=['TaxonomyLevel', 'TaxonomyValue'],
        value_cols=[value_col],
        agg_value_to_label={
            True: 'NewHire',
            False: 'ExistingHire',
        },
    )

    df['Difference'] = df[f'NewHire{value_col}'] - df[f'ExistingHire{value_col}']
    df = df[
        [
            'TaxonomyLevel',
            'TaxonomyValue',
            'Difference',
        ]
    ].drop_duplicates()

    ols_df = get_variable_dataframe(variable, suffix='slopes')

    if significant:
        ols_df = ols_df[
            ols_df['Significant']
        ]

    ols_df = ols_df[
        [
            'TaxonomyLevel',
            'TaxonomyValue',
            'Slope',
            'Significant',
            'P',
        ]
    ].drop_duplicates()

    df = df.merge(
        ols_df,
        on=[
            'TaxonomyLevel',
            'TaxonomyValue',
        ]
    )

    df['YearsToProduceChange'] = df['Difference'] / df['Slope']

    df = df[
        [
            'TaxonomyLevel',
            'TaxonomyValue',
            'YearsToProduceChange',
        ]
    ].drop_duplicates()

    df = df[
        df['YearsToProduceChange'] > 0
    ]

    return df

################################################################################
#
#
# Stats collection
#
#
################################################################################
STATS_COLLECTION = {
    # to replace with stats calls:
    # - get_steepness_by_taxonomy
    'basics': {
        'faculty-hiring-network': get_faculty_hiring_network,
        'closedness-faculty-hiring-network': {
            'kwargs': {'use_closedness_data': True},
            'df': get_faculty_hiring_network,
            'by-gender': {
                'kwargs': {'by_gender': True},
                'do': get_faculty_hiring_network,
            },
        },
        'production': get_faculty_production,
        'hiring': get_faculty_hiring,
        'lorenz': get_lorenz_curves,
    },
    'demographics': {
        'degree-years': degree_years_by_taxonomy,
        'non-doctorates-by-taxonomy': usfhn.demographics.get_non_doctorate_fraction_by_taxonomy,
    },
    'closedness': {
        'df': usfhn.closedness._get_closednesses,
    },
    'taxonomy': {
        'df': usfhn.taxonomy.get_taxonomy_size,
        'institutions': {
            'kwargs': {'count_col': 'InstitutionId'},
            'do': usfhn.taxonomy.get_taxonomy_size,
        },
        'by-year': {
            'kwargs': {'by_year': True},
            'df': usfhn.taxonomy.get_taxonomy_size,
            'institutions': {
                'kwargs': {'count_col': 'InstitutionId'},
                'do': usfhn.taxonomy.get_taxonomy_size,
            },
        },
    },
    'ginis': {
        'kwargs': {'variable': 'ginis'},
        'df': usfhn.gini.get_gini_coefficients,
        'slopes': run_ols_and_get_slopes,
        'stasis': get_years_until_like_new_hires,
        'by-year': {
            'kwargs': {'by_year': True},
            'df': usfhn.gini.get_gini_coefficients,
        },
        'by-faculty-rank': {
            'kwargs': {'by_faculty_rank': True},
            'df': usfhn.gini.get_gini_coefficients,
            'slopes': run_ols_and_get_slopes,
            'by-year': {
                'kwargs': {'by_year': True},
                'df': usfhn.gini.get_gini_coefficients,
            },
        },
        'by-seniority': {
            'kwargs': {'by_seniority': True},
            'df': usfhn.gini.get_gini_coefficients,
            'slopes': run_ols_and_get_slopes,
            'by-year': {
                'kwargs': {'by_year': True},
                'df': usfhn.gini.get_gini_coefficients,
            },
        },
        'by-new-hire': {
            'kwargs': {'by_new_hire': True},
            'df': usfhn.gini.get_gini_coefficients,
            'slopes': run_ols_and_get_slopes,
            'by-year': {
                'kwargs': {'by_year': True},
                'df': usfhn.gini.get_gini_coefficients,
            },
        },
    },
    'gender': {
        'kwargs': {'variable': 'gender'},
        'df': usfhn.gender.get_taxonomy_gender_fraction,
        'slopes': run_ols_and_get_slopes,
        'stasis': get_years_until_like_new_hires,
        'by-year': {
            'kwargs': {'by_year': True},
            'df': usfhn.gender.get_taxonomy_gender_fraction,
        },
        'by-faculty-rank': {
            'kwargs': {'by_faculty_rank': True},
            'df': usfhn.gender.get_taxonomy_gender_fraction,
            'slopes': run_ols_and_get_slopes,
            'by-year': {
                'kwargs': {'by_year': True},
                'df': usfhn.gender.get_taxonomy_gender_fraction,
            },
        },
        'by-seniority': {
            'kwargs': {'by_seniority': True},
            'df': usfhn.gender.get_taxonomy_gender_fraction,
            'slopes': run_ols_and_get_slopes,
            'by-year': {
                'kwargs': {'by_year': True},
                'df': usfhn.gender.get_taxonomy_gender_fraction,
            },
        },
        'by-new-hire': {
            'kwargs': {'by_new_hire': True},
            'df': usfhn.gender.get_taxonomy_gender_fraction,
            'slopes': run_ols_and_get_slopes,
            'by-year': {
                'kwargs': {'by_year': True},
                'df': usfhn.gender.get_taxonomy_gender_fraction,
            },
            'by-rank': {
                'directory_expansions': RANKS_EXPANDER,
                'logits': usfhn.gender.rank_vs_gender_logit,
                'existing-hire-logits': {
                    'kwargs': {'existing_hires': True},
                    'do': usfhn.gender.rank_vs_gender_logit,
                },
            },
        },
        'by-non-us': {
            'kwargs': {'by_non_us': True},
            'df': usfhn.gender.get_taxonomy_gender_fraction,
        },
        'by-career-age': {
            'kwargs': {'by_career_age': True},
            'df': usfhn.gender.get_taxonomy_gender_fraction,
        },
        'by-rank': {
            'directory_expansions': RANKS_EXPANDER,
            'logits': usfhn.gender.rank_vs_gender_logit,
        },
    },
    'self-hires': {
        'kwargs': {'variable': 'self-hires'},
        'df': usfhn.self_hiring.get_taxonomy_self_hiring,
        'slopes': run_ols_and_get_slopes,
        'stasis': get_years_until_like_new_hires,
        'by-year': {
            'kwargs': {'by_year': True},
            'df': usfhn.self_hiring.get_taxonomy_self_hiring,
        },
        'by-faculty-rank': {
            'kwargs': {'by_faculty_rank': True},
            'df': usfhn.self_hiring.get_taxonomy_self_hiring,
            'slopes': run_ols_and_get_slopes,
            'by-year': {
                'kwargs': {'by_year': True},
                'df': usfhn.self_hiring.get_taxonomy_self_hiring,
            },
        },
        'by-gender': {
            'kwargs': {'by_gender': True},
            'df': usfhn.self_hiring.get_taxonomy_self_hiring,
        },
        'by-seniority': {
            'kwargs': {'by_seniority': True},
            'df': usfhn.self_hiring.get_taxonomy_self_hiring,
            'slopes': run_ols_and_get_slopes,
            'by-year': {
                'kwargs': {'by_year': True},
                'df': usfhn.self_hiring.get_taxonomy_self_hiring,
            },
        },
        'by-new-hire': {
            'kwargs': {'by_new_hire': True},
            'df': usfhn.self_hiring.get_taxonomy_self_hiring,
            'slopes': run_ols_and_get_slopes,
            'by-year': {
                'kwargs': {'by_year': True},
                'df': usfhn.self_hiring.get_taxonomy_self_hiring,
            },
            'by-rank': {
                'directory_expansions': RANKS_EXPANDER,
                'logits': usfhn.self_hiring.rank_vs_self_hires_logit,
                'existing-hire-logits': {
                    'kwargs': {'existing_hires': True},
                    'do': usfhn.self_hiring.rank_vs_self_hires_logit,
                },
            },
        },
        'by-career-age': {
            'kwargs': {'by_career_age': True},
            'df': usfhn.self_hiring.get_taxonomy_self_hiring,
        },
        'by-institution': {
            'kwargs': {'by_institution': True},
            'df': usfhn.self_hiring.get_taxonomy_self_hiring,
        },
        'by-rank': {
            'logits': {
                'directory_expansions': RANKS_EXPANDER,
                'do': usfhn.self_hiring.rank_vs_self_hires_logit,
            },
            'binned-logits': {
                'directory_expansions': ALL_RANKS_EXPANDER,
                'do': usfhn.self_hiring.self_hiring_by_career_age_logistics,
            },
        },
    },
    'non-us': {
        'df': usfhn.non_us.get_fraction_non_us,
        'by-year': {
            'kwargs': {'by_year': True},
            'df': usfhn.non_us.get_fraction_non_us,
        },
        'by-institution': {
            'df': usfhn.non_us.get_fraction_non_us_by_institution,
            'by-rank': {
                'directory_expansions': RANKS_EXPANDER,
                'df': usfhn.non_us.get_fraction_non_us_by_institution_with_rank,
                'logits': usfhn.non_us.rank_vs_non_us_logit,
            },
        },
        'by-new-hire': {
            'kwargs': {'by_new_hire': True},
            'by-rank': {
                'directory_expansions': RANKS_EXPANDER,
                'logits': usfhn.non_us.rank_vs_non_us_logit,
                'existing-hire-logits': {
                    'kwargs': {'existing_hires': True},
                    'do': usfhn.non_us.rank_vs_non_us_logit,
                },
            },
        },
        'by-career-age': {
            'kwargs': {'by_career_age': True},
            'df': usfhn.non_us.get_fraction_non_us,
            'by-continent': {
                'kwargs': {'by_continent': True},
                'df': usfhn.non_us.get_fraction_non_us_by_continent_and_career_age,
            },
        },
        'by-country': usfhn.non_us.non_us_countries_by_production,
        'by-continent': usfhn.non_us.non_us_continents_by_production,
        'production': {
            'kwargs': {'non_us': True},
            'do': get_faculty_production,
        },
    },
    'ranks': {
        # we run a lot of the same stats for ranks in different things, so put
        # them into a function. functions:
        # - df
        # - change
        # - hierarchy-stats
        # - mean-change
        **get_rank_stats_functions(),
        'institution-rank-correlations': {
            'directory_expansions': RANKS_EXPANDER,
            'do': usfhn.rank.get_field_to_field_rank_correlations,
        },
        'by-year': {
            'kwargs': {'by_year': True},
            **get_rank_stats_functions(),
        },
        'placements': usfhn.rank.get_placements,
        'institution-fields': usfhn.rank.get_institution_fields,
        'institution-placement-stats': usfhn.rank.get_institution_placement_stats,
    },
    'interdisciplinarity': {
        'df': usfhn.interdisciplinarity.get_employment_interdisciplinarity_df,
        'taxonomy': usfhn.interdisciplinarity.get_taxonomy_interdisciplinarity,
        'institution': usfhn.interdisciplinarity.get_institution_interdisciplinarity,
        'degree-institution': usfhn.interdisciplinarity.get_degree_institution_interdisciplinarity,
        'by-year': {
            'kwargs': {'by_year': True},
            'df': usfhn.interdisciplinarity.get_employment_interdisciplinarity_df,
            'taxonomy': usfhn.interdisciplinarity.get_taxonomy_interdisciplinarity,
        },
        'institution-with-ranks': {
            'prefixes': RANKS_EXPANDER,
            'do': usfhn.interdisciplinarity.institutions_with_ranks,
        },
        'by-rank': {
            'directory_expansions': RANKS_EXPANDER,
            'logits': usfhn.interdisciplinarity.rank_vs_interdisciplinarity_logit,
        },
    },
    'careers': {
        'df': usfhn.careers.get_career_moves_df,
        'gender': usfhn.careers.get_move_risk_by_gender, # gender-overall
        'by-year': {
            'gender': usfhn.careers.get_move_risk_by_gender_and_year, # gender-by-year
        },
        'institution-risk': usfhn.careers.get_institution_move_risk,
        'taxonomy-hires-before-and-after-mcms': usfhn.careers.get_taxonomy_hires_before_and_after_mcms,
        'hierarchy-changes-from-mcms': usfhn.careers.get_mcm_stats,
        'hierarchy-changes-from-mcms-compare-to-normal': usfhn.careers.get_mcm_stats_comparison,
        'length': {
            'df': usfhn.careers.get_career_lengths_df,
            'taxonomy': usfhn.careers.career_length_by_taxonomy,
            'gender': usfhn.careers.career_length_by_gender,
            'gini': usfhn.careers.get_gini_coefficients_by_career_length,
            'by-year': {
                'kwargs': {'by_year': True},
                'df': usfhn.careers.get_career_lengths_df,
                'gender': usfhn.careers.career_length_by_gender,
                'taxonomy': usfhn.careers.career_length_by_taxonomy,
                'gini': usfhn.careers.get_gini_coefficients_by_career_length,
            },
            'by-rank': {
                'directory_expansions': RANKS_EXPANDER,
                'logits': usfhn.careers.rank_vs_career_length_logit,
            },
        },
    },
    'new-hire': {
        'df': usfhn.new_hires.get_new_hires_df,
        'self-hires': usfhn.new_hires.get_self_hires,
        'non-us': usfhn.non_us.get_new_hire_fraction,
        'interdisciplinarity': usfhn.interdisciplinarity.get_new_hire_fraction,
        'steepness': usfhn.new_hires.get_steepness,
        'production': {
            'kwargs': {'new_hires': True},
            'do': get_faculty_production,
        },
        'lorenz': {
            'kwargs': {'new_hires': True},
            'do': get_lorenz_curves,
        },
        'by-year': {
            'kwargs': {'by_year': True},
            'df': usfhn.new_hires.get_new_hires_df,
            'self-hires': usfhn.new_hires.get_self_hires,
            'non-us': usfhn.non_us.get_new_hire_fraction,
            'interdisciplinarity': usfhn.interdisciplinarity.get_new_hire_fraction,
            'steepness': usfhn.new_hires.get_steepness,
        },
    },
    'non-new-hire': {
        'production': {
            'kwargs': {'non_new_hires': True},
            'do': get_faculty_production,
        },
        'lorenz': {
            'kwargs': {'non_new_hires': True},
            'do': get_lorenz_curves,
        },
    },
    'non-doctorate': {
        'df': usfhn.non_doctorate.get_non_doctorates_by_taxonomy,
        'by-year': {
            'kwargs': {'by_year': True},
            'df': usfhn.non_doctorate.get_non_doctorates_by_taxonomy,
        },
    },
    'attrition': {
        'df': usfhn.attrition.get_attritions_df,
        'by-gender': {
            'age-of-attrition': usfhn.attrition.mean_age_of_attrition,
        },
        'by-self-hire': {
            'risk-ratio': usfhn.attrition.self_hire_to_non_self_hire_risk_ratio,
            'prestige-deciles': {
                'directory_expansions': ALL_RANKS_EXPANDER,
                'do': usfhn.self_hiring.self_hiring_binned_by_prestige_attrition_risk,
            },
        },
        'risk': {
            # for risk, we run a LOT of different risk combinations, so I put
            # them in a function. The keys are:
            # - taxonomy
            # - gender
            # - self-hires
            # - institution, which is expanded over:
            #   - institution_column: ['DegreeInstitutionId', 'InstitutionId']
            **get_attrition_risk_stats_functions(),
            'by-career-stage': {
                'kwargs': {'by_career_stage': True},
                **get_attrition_risk_stats_functions(),
            },
            'by-time': {
                'kwargs': {'by_year': True},
                **get_attrition_risk_stats_functions(),
                'by-career-stage': {
                    'kwargs': {'by_career_stage': True},
                    **get_attrition_risk_stats_functions(),
                },
                'by-degree-year': {
                    'kwargs': {'by_degree_year': True},
                    **get_attrition_risk_stats_functions(),
                },
            },
            'by-degree-year': {
                'kwargs': {'by_degree_year': True},
                **get_attrition_risk_stats_functions(),
            },
            'gender-balanced-hiring': usfhn.attrition.get_gender_balance_given_attrition_and_balanced_hiring,
        },
        'by-rank': {
            'directory_expansions': ALL_RANKS_EXPANDER,
            'df': usfhn.attrition.attritions_df_with_ranks,
            'institution': {
                'risk': usfhn.attrition.institutions_with_ranks,
                'logits': usfhn.attrition.rank_vs_attrition_risk_logit,
                'gendered-logits': {
                    'kwargs': {'by_gender': True},
                    'do': usfhn.attrition.rank_vs_attrition_risk_logit,
                },
                'by-self-hire': {
                    'kwargs': {'by_self_hire': True},
                    'logits': usfhn.attrition.rank_vs_attrition_risk_logit,
                },
            },
            'academia-ranks': {
                'kwargs': {'academia_rank_only': True},
                'df': usfhn.attrition.attritions_df_with_ranks,
                'institution': {
                    'logits': usfhn.attrition.rank_vs_attrition_risk_logit,
                },
            }
        },
    },
    'pool-reduction': {
        'umbrella-stats': usfhn.pool_reduction.umbrella_exclusion_stats,
        'excluded-taxonomy-sizes': usfhn.pool_reduction.excluded_taxonomy_sizes,
        'excluded-at-field-level': {
            'do': usfhn.pool_reduction.people_excluded_at_field_level_by_umbrella,
            'prefix_expansions': {'dataset': ['unfiltered-census-pool', 'alternate-fields']},
        },
        'excluded-at-umbrella-level': {
            'do': usfhn.pool_reduction.people_excluded_at_domain_level_by_umbrella,
            'prefix_expansions': {'dataset': ['unfiltered-census-pool', 'alternate-fields']},
        },
        'paper-stats': {
            'expansion_type': hnelib.runner.JSONExpansion,
            'general': usfhn.pool_reduction.get_stats,
        },
    },
    'geodesics': {
        'df': usfhn.webs.get_geodesics_df,
    },
}

runner = hnelib.runner.DataFrameRunner(
    collection=STATS_COLLECTION,
    directory=usfhn.constants.STATS_PATH,
)

if __name__ == '__main__':
    parser = argparse.ArgumentParser(description='stat runner')
    parser.add_argument('--which', '-w', type=str, help='which thing to run')
    parser.add_argument('--all', default=False, action='store_true')
    parser.add_argument('--run_expansions', '-e', default=False, action='store_true', help='run combinations')
    parser.add_argument('--clean', default=False, action='store_true', help='clean old stats')
    args = parser.parse_args()

    if args.clean:
        runner.clean()

    to_run = []
    if args.which:
        to_run.append(args.which)
    elif args.all:
        to_run = list(runner.collection.keys())

    for item_name in to_run:
        runner.run(item_name, run_expansions=args.run_expansions)