#!/usr/bin/env python
# -*- coding:UTF-8 -*-

'''
    Python challenge: level 4
        urllib may help. DON’T TRY ALL NOTHINGS, since it will never
        end. 400 times is more than enough.

    @author: DTree
    @license: GNU General Public License 2.0 or later
    @contact: mycodekingdom@gmail.com
'''

from __future__ import print_function
import urllib

if __name__ == "__main__":
    url = "http://www.pythonchallenge.com/pc/def/linkedlist.php?nothing="
    num = "12345"
    i = 0
    while i < 400:
        request = urllib.urlopen(url+num)
        text = request.read()
        print(text)
        if text.find(".html") != -1:
            break

        try:
            num = text[text.index("and the next nothing is")+24:]
        except ValueError:
            num = str(int(num)/2)

        i += 1

    print(text)


from tkinter import *

def button_clicked():
    new_text = float(entry.get()) * 1.609344
    km_converted.config(text=new_text)

window = Tk()
window.title("Mile to Km Converter")
window.config(padx=10, pady=10)

entry = Entry(width=5)
entry.insert(END, string="0")
print(entry.get())
entry.grid(column=1,row=0)

miles_label = Label(text="Miles", font=("Arial", 12, "bold"))
miles_label.grid(column=2, row=0)

km_converted = Label(text="0", font=("Arial", 12, "bold"))
km_converted.grid(column=1, row=1)

equal_label = Label(text="is equal to", font=("Arial", 12, "bold"))
equal_label.grid(column=0, row=1)

km_label = Label(text="Km", font=("Arial", 12, "bold"))
km_label.grid(column=2, row=1)

button = Button(text="Calculate", command=button_clicked)
button.grid(column=1, row=2)

window.mainloop()


# this file holds the functionality for the to-do interface, inside a notebook framework which may be scaled to include
# the main note-taking app in another notebook tab.

from tkinter import (
    Tk,
    Frame,
    Text,
    Button,
    Checkbutton,
    Entry,
    Label,
    Listbox,
    END,
    Menu,
)
from tkinter.ttk import Notebook
import json
from datetime import datetime

# button labels as constants
ADD_NEW = "^ Enter & Add TODO ^"
ARCHIVE = "^ Archive Entry v"
DEL_ENTRY = "-Remove Archive-"

# reference file to save and restore state of listbox between instances, using JSON.
FILE1 = "todo_active"
FILE2 = "todo_archive"

class NotebookMain(Tk):
    def __init__(self, parent, *args, **kwargs):
        self.parent = parent
        self.window = Frame(self.parent)

        def notebook_assets(self):
            self.notebook1 = Notebook(
                self.parent)
            self.page1 = Frame(self.notebook1, bg="#000000")
            #self.page2 = Frame(self.notebook1)
            self.notebook1.add(self.page1, text="TODO")
            #self.notebook1.add(self.page2, text="NOTES")
            self.notebook1.pack()

            def todo_screen():
                def column1():
                    def add_entry(event):
                        todo = entry_1.get()
                        active_lbox.insert(
                            "end", todo)

                    def archive_entry(event):
                        selection = active_lbox.curselection()[0]
                        grabber = active_lbox.get(selection)
                        archive_lbox.insert("end", grabber)
                        active_lbox.delete(selection)

                    def de_archive_entry(event):
                        selection = archive_lbox.curselection()[0]
                        grabber = archive_lbox.get(selection)
                        active_lbox.insert("end", grabber)
                        archive_lbox.delete(selection)

                    def delete_active(event):
                        selection = active_lbox.curselection()[0]
                        active_lbox.delete(selection)

                    def delete_archive(event):
                        selection = archive_lbox.curselection()[0]
                        archive_lbox.delete(selection)

                    def save_state(event):
                        note = active_lbox.get(0, "end")
                        note2 = archive_lbox.get(0, "end")
                        with open(FILE1, "w") as f:
                            json.dump(note, f)
                            # json.dump(note2, f)
                        with open(FILE2, "w") as f2:
                            json.dump(note2, f2)
                        dt = datetime.now()
                        timestamp = f"{dt.hour}:{dt.minute}"
                        save_confirm.delete("1.0", "end")
                        save_confirm.insert("1.0", "TODO saved: " + str(timestamp))

                    def load_state():
                        with open(FILE1, "r+") as f:
                            data = json.load(f)
                            for v in data:
                                active_lbox.insert(END, v)
                        with open(FILE2, "r+") as f2:
                            data = json.load(f2)
                            for v in data:
                                archive_lbox.insert(END, v)

                    entry_1 = Entry(self.page1, highlightbackground="#9E9E9E", highlightthickness=1)
                    entry_1.grid(row=2, column=1, ipadx=20, pady=3)
                    entry_1.bind("<Tab>", add_entry)

                    active_lbox = Listbox(self.page1, width=18, highlightbackground="#9E9E9E",
                                          highlightthickness=1, font=("RobotoRoman", 12))
                    active_lbox.grid(row=4, column=1, pady=3)
                    active_lbox.bind("<Tab>", archive_entry)
                    active_lbox.bind("<Control-s>", save_state)
                    active_lbox.bind("<Delete>", delete_active)

                    archive_lbox = Listbox(self.page1, height=3, width=27, highlightbackground="#9E9E9E",
                                           highlightthickness=1, fg="#606060")
                    archive_lbox.grid(row=6, column=1, pady=3)
                    archive_lbox.bind("<Tab>", de_archive_entry)
                    archive_lbox.bind("<Delete>", delete_archive)
                    archive_lbox.bind("<Control-s>", save_state)

                    add_entry_button = Button(self.page1, text=ADD_NEW, command=lambda: add_entry(entry_1), width=20,
                                              relief="raised")
                    add_entry_button.grid(row=3, column=1, pady=3)

                    archive_entry_button = Button(self.page1, text=ARCHIVE, command=lambda: archive_entry(active_lbox),
                                                  width=20, relief="raised")
                    archive_entry_button.grid(row=5, column=1, pady=3)

                    delete_archived_button = Button(
                        self.page1, text=DEL_ENTRY, command=lambda: delete_archive(archive_lbox),
                        width=20, relief="raised")
                    delete_archived_button.grid(row=7, column=1, pady=3)

                    save_button = Button(self.page1, text="SAVE TODO", command=lambda: save_state(self.window))
                    save_button.grid(row=9, column=1)
                    save_button.config(font=("RobotoRoman", 11, "bold"), width=20, bg="#8E001C", fg="#FFB302")

                    save_confirm = Text(self.page1, height=1, width=23)
                    save_confirm.grid(row=8, column=1, pady=2)

                    try:
                        load_state()
                    except FileNotFoundError:
                        pass

                column1()

            todo_screen()

            def note_screen():
                pass

        notebook_assets(self)

def run():
    root = Tk()
    app = NotebookMain(root)
    root.mainloop()

# running here and on main.py resulted in break
# if __name__ == "__main__":
#     run()


import torch
import pickle
from dataloader import LymphocytesFlat
from torch.utils.data import DataLoader
from utils import load_attention_mil
import numpy as np
from MIL import MILModel
import pandas as pd

device = torch.device('cpu')
dataset_train = LymphocytesFlat('trainset')
dataset_test = LymphocytesFlat('testset')

dataloader_train = DataLoader(dataset_train, batch_size=3, shuffle=True, num_workers=3)
dataloader_test = DataLoader(dataset_test, batch_size=3, shuffle=True, num_workers=3)

autoencoder_feature_extractor = torch.load('trained_models/autoencoder.ckpt', map_location=device).encoder
instance_predictor = None
bag_predictor = load_attention_mil('trained_models/attention_deep_pooling_mil_1.h5', 60, 198)

mil_model = MILModel(autoencoder_feature_extractor, None, bag_predictor)
prediction_train = mil_model.predict(dataloader_train)
prediction_test = mil_model.predict(dataloader_test)

df_train = pd.DataFrame(prediction_train, columns = ['ID', 'Proba'])
df_test = pd.DataFrame(prediction_test, columns = ['ID', 'Proba'])
df_all = df_train.append(df_test)
df_all['ID'] = df_all['ID'].apply(lambda x: f'P{int(x)}')
annotation_df = pd.read_csv('clinical_annotation.csv')
merged_df = annotation_df.merge(df_all, on='ID')
print(merged_df)
merged_df.to_csv('clinical_and_proba.csv', index= False)


#!/usr/bin/env python

import smtplib
from email.message import EmailMessage

me = "cmeafestivalcoordinator@gmail.com"
you = "durbin.james@gmail.com"

server = smtplib.SMTP('smtp.gmail.com',587) #port 465 or 587
# server.ehlo()
server.starttls()
# server.ehlo()
server.login(me,'qkqwesilxpfnwlwe')

msg = "Message_you_need_to_send"

server.sendmail(me,you,msg)
server.close()


import os
import re
import argparse
import logging
import json

logger = logging.getLogger(__name__)
logger.setLevel(logging.DEBUG)
ch = logging.StreamHandler()
ch.setLevel(logging.DEBUG)
formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
ch.setFormatter(formatter)
logger.addHandler(ch)

parser = argparse.ArgumentParser()
parser.add_argument("--task_type", type=str, default="s_cls", help="s_cls, p_cls, seq_tag")
parser.add_argument("--task_name", type=str, default="mtop", help="mtop, m_atis")
parser.add_argument("--in_dir", type=str, help="task's root dir that contains lang-spec subdirs")
parser.add_argument("--out_dir", type=str, help="")
args = parser.parse_args()

TASK_EXT = {
    "mtop": {
      "s_cls": [".txt"],
      "seq_tag": [".txt"],
      },
    "m_atis": {
      "s_cls": [".txt"],
      "seq_tag": [".txt"],
      },
    }

TASK_DIM_RAW = {
    "mtop": {
      "s_cls": "\t",
      "seq_tag": "\t",
      },
    "m_atis": {
      "s_cls": "\t",
      "seq_tag": "\t",
      },
    }

LABEL_DIM = {
    "mtop": {
      "s_cls": "",
      "seq_tag": ",",
      },
    "m_atis": {
      "s_cls": "#| ", # FIXME: shouldn't contain multi-labels, but in fact the raw data contains
      "seq_tag": "-",
      },
    }

SET_TYPE_NORM = {
    "test": "test",
    "dev": "dev",
    "eval": "dev",
    "train": "train",
    }

TASKS_WITH_HEAD = ["m_atis"]

def parse_raw(in_file, task_name, task_type):
  examples = []
  dim = TASK_DIM_RAW[task_name][task_type]
  with open(in_file, 'r') as fin:
    for i, line in enumerate(fin):
      if task_name in TASKS_WITH_HEAD and i < 1: continue
      ex = {}
      fields = line.strip().split(dim)
      if task_name == "mtop":
        ex["id"], ex["intents"], ex["slots"], ex["sent"], ex["domain"], ex["lang"], ex["decoupled_form"], ex["tokens"] = fields
      elif task_name == "m_atis":
        ex["id"], ex["sent"], ex["slots"], ex["intents"] = fields
      examples.append(ex)
  return examples

def format_tsv(examples, task_name, task_type, out_file):
  logger.info(f" [format_tsv] formating {out_file}, {len(examples)} exs")
  file_labels = set()
  with open(out_file, 'w') as fout:
    if task_type == "s_cls":
      fout.write("sent\tlabel\n")
    elif task_type == "p_cls":
      pass
    elif task_type == "seq_tag":
      pass

    if task_type == "seq_tag":
      fidx = open(out_file + ".idx", 'w')

    for idx, ex in enumerate(examples):
      if task_type == "s_cls":
        fout.write("{}\t{}\n".format(ex["sent"], ex["intents"]))
        if LABEL_DIM[task_name][task_type]:
          # file_labels.update(re.split(LABEL_DIM[task_name][task_type], ex["intents"]))
          file_labels.add(re.split(LABEL_DIM[task_name][task_type], ex["intents"])[0]) # NOTE: should be 1-label classification, only take the first label
        else:
          file_labels.add(ex["intents"])
      elif task_type == "p_cls":
        pass
      elif task_type == "seq_tag":
        if task_name == "mtop":
          # NOTE: `sent` is not tokenized, instead `tokens` is
          # we use `tokens` since it's aligned with `slots` labels
          try:
            tokens_raw = json.loads(ex["tokens"])
          except json.decoder.JSONDecodeError:
            logger.warning(ex["tokens"])
            tokens_raw = json.loads(ex["tokens"].replace('"""', '"\\""'))
            # raise
          tokens, token_spans = tokens_raw["tokens"], tokens_raw["tokenSpans"]
          labels_raw = ex["slots"].split(',')
          labels = ['O'] * len(tokens)
          offset2label = {}
          for label_raw in labels_raw:
            if not label_raw.strip(): continue # skip null label_raw
            try:
              start, end, _, label = label_raw.split(':') # fmt: start_byte:end_byte:SL:slot_name
            except ValueError:
              logger.error(f" [format_tsv] ex = {ex}")
              logger.error(f" [format_tsv] label_raw = {label_raw}")
              raise
            for offset in range(int(start), int(end)):
              offset2label[offset] = label
          for tok_i, span in enumerate(token_spans):
            if span["start"] in offset2label:
              labels[tok_i] = offset2label[span["start"]]
        elif task_name == "m_atis":
          tokens = ex["sent"].split(' ') # NOTE: zh `sent` is tokenized
          labels = ex["slots"].split(' ')
          # for word, label in zip(ex["sent"].split(' '), ex["slots"].split(' ')):

        for l_i in range(len(labels)): # replace null labels as 'O'
          if not labels[l_i].strip():
            labels[l_i] = 'O'
        for word, label in zip(tokens, labels):
          fout.write(f"{word}\t{label}\n") # NOTE: did NOT distinguish B-/I-
          fidx.write(f"{idx}\n")
        fout.write(f"\n")
        fidx.write(f"\n")
        file_labels = file_labels.union(labels)

    if task_type == "seq_tag":
      fidx.close()

  logger.info(f" [format_tsv] done. {len(file_labels)} file_labels: {file_labels}")
  return file_labels

def main():
  all_labels = set()
  for p_dir, sub_dirs, _ in os.walk(args.in_dir):
    for lang in sub_dirs:
      lang_labels = set()
      for pp_dir, _, files in os.walk(os.path.join(p_dir, lang)):
        if args.task_name == "mtop":
          in_files = [os.path.join(pp_dir, filename) for filename in files if os.path.splitext(filename)[-1] in TASK_EXT[args.task_name][args.task_type]]
        elif args.task_name == "m_atis":
          in_files = [os.path.join(pp_dir, filename) for filename in files if os.path.splitext(filename)[-1] in TASK_EXT[args.task_name][args.task_type]]
        else:
          raise NotImplementedError(f" unknown args.task_name {args.task_name}")
        for in_file in in_files:
          examples = parse_raw(in_file, args.task_name, args.task_type)
          set_type = ""
          for ftype in SET_TYPE_NORM:
            if ftype in os.path.basename(in_file):
              set_type = SET_TYPE_NORM[ftype]
              break
          work_dir = os.path.join(args.out_dir, f"{args.task_name}-{args.task_type}", set_type)
          os.makedirs(work_dir, exist_ok=True)
          out_file = os.path.join(work_dir, lang)
          lang_labels.update(format_tsv(examples, args.task_name, args.task_type, out_file))
        break
      logger.info(f" [format_tsv] {lang}: {len(lang_labels)} lang_labels: {lang_labels}")
      all_labels.update(lang_labels)
    break
  logger.info(f" [format_tsv] {len(all_labels)} all_labels: {all_labels}")

if __name__ == "__main__":
  main()


'''Reference for google auth: https://developers.google.com/identity/protocols/oauth2/web-server
    any other reference for this process in google's documentation is outdated
    google now uses it's own authentication libraries and not third party libs like Oauth
    So don't trust the top links that come up in a google search.
'''

from email import message
import os
from re import search
from django.http.response import HttpResponse
from django.shortcuts import render
from django.shortcuts import redirect
from django.core import files

import requests
import google.oauth2.credentials
import google_auth_oauthlib.flow
import googleapiclient.discovery
import httplib2
import email
import base64
from bs4 import BeautifulSoup, SoupStrainer
import pprint
import quopri
import html
from urllib import parse
from nltk.corpus import wordnet, stopwords
from nltk import word_tokenize
from nltk.util import ngrams
from read_email.models import User, ClosetItem, Closet
import re
from io import BytesIO
from fake_useragent import UserAgent
from selenium import webdriver
from selenium.webdriver.chrome.options import Options
from webdriver_manager.chrome import ChromeDriverManager
from PIL import Image
# This variable specifies the name of a file that contains the OAuth 2.0
# information for this application, including its client_id and client_secret.
CLIENT_SECRETS_FILE = "client_secret.json"

# This OAuth 2.0 access scope allows for full read/write access to the
# authenticated user's account and requires requests to use an SSL connection.
SCOPES = ['https://www.googleapis.com/auth/gmail.readonly', 'https://www.googleapis.com/auth/userinfo.email', 'https://www.googleapis.com/auth/userinfo.profile', 'openid']
API_SERVICE_NAME = 'gmail'
API_VERSION = 'v1'
REDIRECT_URI = 'http://localhost:8000/oauth2callback'
TARGET_BRANDS = ['acne studio', 'miranda', 'lilo', 'lille', 'nora', 'judith', 'chocolat', 'urbanita', 'high']
STOP_WORDS = ['address', 'delivery', 'exchange', 'what', 'that', 'chat']
def get_hyponyms(word):
    syn = wordnet.synsets(word)[0]
    hypos = lambda s:s.hyponyms()
    closure = syn.closure(hypos)
    return [s.name().split('.')[0].lower() for s in closure
            if s.name().split('.')[0].lower() != 'livery'
            and s.name().split('.')[0].lower() != 'black']
CLOTHING_HYPONYMS = get_hyponyms('clothing')
for cloth in CLOTHING_HYPONYMS:
    if cloth in 'jeans miranda':
        print('cloth:', cloth)
CLOTHING_HYPONYMS.append('shorts')
CLOTHING_HYPONYMS.append('legwarmers')
CLOTHING_HYPONYMS.append('leg-warmers')
CLOTHING_HYPONYMS.append('short')
CLOTHING_HYPONYMS.append('loafers')
VENDORS = ['saks fifth avenue', 'h&m', 'net-a-porter', 'mango']
credentials_obj = None

####HELPERS#####

def simulate_browser(url):
    # Instantiate an Options object
# and add the "--headless" argument
    #opts = Options()
    #opts.add_argument(" --headless")
    #opts.add_argument('User-Agent="Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/96.0.4664.55 Safari/537.36"')
    #opts.add_argument('Accept="text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9"')
    #opts.add_argument('Accept-Language="en-US,en;q=0.9"') .
    # Instantiate a webdriver
    browser = webdriver.Chrome(ChromeDriverManager().install())
    browser.get(url)
    browser.execute_script("return document.getElementsByTagName('html')[0].innerHTML")
    return browser

def write_to_file(text):
    f = open('html.txt', 'w')
    f.writelines(text)
    f.close()

def is_price(text):
    if re.search('(\d+\.\d+)', text):
        return True
    return False

def get_date_from_snippet(snippet):
    date_str = snippet[snippet.find('Date:')+11:]
    date_str = date_str[:date_str.find('at')]
    #print('Date:', date_str)
    return date_str

def get_source_site_from_snippet(snippet):
    source_site = snippet[snippet.find('@'): snippet.find('>')]
    if source_site.count('.') > 1:
        source_site = source_site[source_site.find('.')+1:]
    #print(source_site)
    return source_site

def get_price(text):
    return re.search('(\d+\.\d+)', text).group(0)

def clean_string(html_text):
    html_text = html_text.replace('=','')
    return html.unescape(html_text)

def clean_html(html_text):
    html_text = html_text.replace('\r\n','')
    html_text = html_text.replace('<=', '<')
    html_text = html_text.replace('that', '')

    return html_text

def make_ordered_set(list_):
    return list(dict.fromkeys(list_))

CLOTHING_HYPONYMS = make_ordered_set(CLOTHING_HYPONYMS)

def tokenize(text):
    text.replace('&', '')
    return word_tokenize(text.lower())

def add_to_db(user_attribs, closet_item_attribs):

    if (not User.objects.filter(name=user_attribs['name']).exists()): # If google user does not exist in the DB, update the DB,
        user_entry = User(name=user_attribs['name'], email=user_attribs['email'])
        user_entry.save()
    else: # Else delete and add fresh
        User.objects.get(name=user_attribs['name']).delete()
        user_entry = User(name=user_attribs['name'], email=user_attribs['email'])
        user_entry.save()
    for i in range(len(closet_item_attribs['item_names'])):
        closet_item = ClosetItem(
            item_name = closet_item_attribs['item_names'][i],
            brand = closet_item_attribs['brands'][i],
            purchase_price = closet_item_attribs['purchase_prices'][i],
            vendor = closet_item_attribs['vendor'],
            purchased_date = closet_item_attribs['purchased_date'],
            source_site = closet_item_attribs['source_site']
            #image = closet_item_attribs['image_files'][i]
        )
        if len(closet_item_attribs['image_names']) > 0 or len(closet_item_attribs['image_files'])>0:
            closet_item.image.save(closet_item_attribs['image_names'][i] ,closet_item_attribs['image_files'][i])
        closet_item.save()
        closet = Closet(user=user_entry, closet_item=closet_item)
        closet.save()

def n_grams(text, n):
    n_grams = ngrams(tokenize(text), n)
    return [' '.join(grams) for grams in n_grams]

def get_urls_from_html(html_text):
    soup = BeautifulSoup(html_text, 'html.parser')
    embedded_links = []
    for link in soup.find_all('a'):
        embedded_links.append(link.get('href'))
    return embedded_links

def credentials_to_dict(credentials):
  return {'token': credentials.token,
          'refresh_token': credentials.refresh_token,
          'token_uri': credentials.token_uri,
          'client_id': credentials.client_id,
          'client_secret': credentials.client_secret,
          'scopes': credentials.scopes}

def has_stopword(s):
    for stopword in STOP_WORDS:
        if stopword in s.lower():
            return True
    return False

def has_clothingword(s):
    for w in CLOTHING_HYPONYMS:
        if w in s:
            return True
    return False

def has_itemword(s,item_tkns):
    hasitemword = False
    for item_tkn in item_tkns:
        for w in item_tkn:
            if w != '&':
                if w.lower() in s.lower():
                    #print('w:',w,'s:',s)
                    hasitemword = True
    print(hasitemword)
    return hasitemword

def decode_html(html_txt):
    html_txt = base64.urlsafe_b64decode(html_txt)
    html_txt = quopri.decodestring(html_txt).decode('iso-8859-1')
    html_txt = clean_html(html_txt)
    return html_txt

def search_messages(service, user_id, search_string):
    search_id = service.users().messages().list(userId=user_id, q=search_string).execute()
    #print('list response: ', search_id)
    number_results = search_id['resultSizeEstimate']
    if number_results > 0:
        message_ids = search_id['messages']
        final_list  = []
        for ids in message_ids:
            final_list.append(ids['id'])
        return final_list
    else:
        print('There were 0 results for that search string, returning an empty string')
        return ''

def scrape_product_images(soup, product_links):
    #scrape data from product links from vendor
    product_image_filenames = []
    product_image_files = []

    for link_tag in soup.find_all('a'):
        #print('link_tag:', link_tag)
        for prod_link in product_links:
            #print('lt:',link_tag['href'])
            #print('pl:', prod_link)
            if link_tag['href'] == prod_link:
                img_tag = link_tag.find('img')
                if img_tag:
                    img_src = img_tag['src']
                    if img_src[0] != 'h':
                        img_src = 'https:' + img_src
                    #print('img_src',img_src)
                    response = requests.get(img_src)
                    #print('resstatus:', response.status_code)
                    if response.status_code != requests.codes.ok:
                        print('bad response on img_src')
                    else:
                        fp = BytesIO()
                        try:
                            Image.open(BytesIO(response.content))
                        except OSError:
                            print('Not a valid image')
                        fp.write(response.content)
                        product_image_filename = img_src.split("/")[-1]
                        product_image_file = files.File(fp)
                        product_image_files.append(product_image_file)
                        product_image_filenames.append(product_image_filename)
    return product_image_files, product_image_filenames
        ##print('resContent:', res.content)
        ##html_from_product_link = res.content
        #html_from_product_link = simulate_browser(prod_link)
        ##html_from_product_link = decode_html(html_from_product_link)
        #print('htmlfromProductlink', html_from_product_link)
        #prod_soup = BeautifulSoup(html_from_product_link, "html.parser")
        #write_to_file(str(prod_soup.prettify()))

def decode_message(service, user_id, msg_ids, user_info):
    text_parts = []

    for msg_id in msg_ids:
        message = service.users().messages().get(userId=user_id, id=msg_id, format='raw').execute()
#        message_full = service.users().messages().get(userId=user_id, id=msg_id, format='full').execute()
        #pprint.pp(message_full)
        snippet = html.unescape(message['snippet'])
        #print('snippet:',snippet)

        msg_byte = base64.urlsafe_b64decode(message['raw'].encode('utf-8'))

        msg_str = email.message_from_bytes(msg_byte)
        content_types = msg_str.get_content_maintype()

        if content_types == 'multipart':
            plain_text, html_text = msg_str.get_payload()
            plain_text = plain_text.get_payload()
            html_text  = html_text.get_payload()
            html_text  = quopri.decodestring(html_text).decode('iso-8859-1')
            scrape_brand_item_price(snippet, html_text, user_info)
        else:
            #print('msg:', msg_str.get_payload())
            text_parts.append(msg_str.get_payload())

def scrape_brand_item_price(html_snippet, html_text, user_info):

    purchase_prices = []
    brands = []
    items  = []
    product_links  = []
    vendor = ''
    product_image_file = ''
    product_image_filename = ''
    html_text = clean_html(html_text)
    soup = BeautifulSoup(html_text, 'html.parser')
    purchased_date = get_date_from_snippet(html_snippet)
    source_site = get_source_site_from_snippet(html_snippet)
    for p in soup('p'):
        p.decompose()

    item_tkns = []

    for string in soup.stripped_strings:
        string = clean_string(str(string))

        if len(string) > 60 or has_stopword(string):
            continue

        for item in CLOTHING_HYPONYMS:
            if item in string.lower():
                items.append(string)
                continue

        if is_price(string) and len(purchase_prices) < len(items):
            purchase_prices.append(get_price(string))

        for brand in TARGET_BRANDS:
            if brand in string.lower() and len(brands) < len(items):
                brands.append(brand)
        if not vendor:
            for vend in VENDORS:
                if vend in string.lower():
                    vendor = vend

    #items = make_ordered_set(items)
    #purchase_prices = make_ordered_set(purchase_prices)
    #brands = make_ordered_set(brands)
    #print(items)
    # get links for products in emails
    for link_tag in soup.find_all('a'):
        link_text = link_tag.string
        if link_text:
            if not has_stopword(link_text.lower()):
                if has_clothingword(link_text.lower()):
                    product_links.append(link_tag['href'])
    #print('prodlinks:', product_links)

    product_image_files, product_image_filenames = scrape_product_images(soup, product_links)
    header = { # Do not delete, might be useful later if you need to spoof a web browser.
        'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/96.0.4664.55 Safari/537.36',
        'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9'
        }
    make_ordered_set(product_image_files)
    make_ordered_set(product_image_filenames)
    #print(res.content)
    if len(brands) == 0:
        for i in range(len(items)):
            brands.append('select a brand')
    user_attribs = {
        'name' : user_info['name'],
        'email' : user_info['email']
    }
    closet_item_attribs = {
        'item_names' : items,
        'vendor' : vendor,
        'purchase_prices': purchase_prices,
        'purchased_date': purchased_date,
        'source_site' : source_site,
        'brands' : brands,
        'image_names' : product_image_filenames,
        'image_files' : product_image_files
    }
    #print('brands: ', brands)
    #print('items:', items)
    #print('prices:', prices)
    pprint.pp(closet_item_attribs)
    add_to_db(user_attribs, closet_item_attribs)
#############################################################

####VIEWS#####

def test_api_request_view(request):
    if 'credentials' not in request.session:
        return redirect('authorize/')
      # Load credentials from the session.
    credentials = google.oauth2.credentials.Credentials(
      **request.session['credentials'])

    service = googleapiclient.discovery.build(
        API_SERVICE_NAME, API_VERSION, credentials=credentials
    ) # Use service object to do things with emails

    # Get user info: name, email, etc.
    user_info = requests.get(
        'https://www.googleapis.com/oauth2/v2/userinfo',
        headers={'Authorization':'Bearer ' + credentials.token}).json() # json() Makes dict from json
    #add_user_to_db(user_info) # add user record to DB
    #print('userinfo:',user_info)
    saks_subj =  'subject:Thank You for Your Order #287529239'
    mango_subj = 'subject:Thank you for shopping at MANGO'
    net_a_porter_subj = 'subject: Your NET-A-PORTER order confirmation'
    handm_subj = 'subject: H&M Order Confirmation'
    thnks_subj = 'subject:Thank You'
    uniqlo_subj = 'subject:Your UNIQLO order has shipped!'
    all_receipts = 'category:purchases | subject:your order '
    message_ids = search_messages(service, 'me', all_receipts)
    #print('len(msg_ids', len(message_ids))
    User.objects.all().delete()
    ClosetItem.objects.all().delete()
    Closet.objects.all().delete()
    decode_message(service, 'me', message_ids, user_info)
    return HttpResponse('msgs[0]')

def authorize_view(request):
    flow = google_auth_oauthlib.flow.Flow.from_client_secrets_file(
        CLIENT_SECRETS_FILE, scopes=SCOPES
    )
    flow.redirect_uri = REDIRECT_URI

    authorization_url, state = flow.authorization_url(
      # Enable offline access so that you can refresh an access token without
      # re-prompting the user for permission. Recommended for web server apps.
      access_type='offline',
      # Enable incremental authorization. Recommended as a best practice.
      include_granted_scopes='true')

    request.session['state'] = state
    return redirect(authorization_url)

def oauth2callback_view(request):
    # Specify the state when creating the flow in the callback so that it can
    # verified in the authorization server response.
    state = request.session['state']

    flow = google_auth_oauthlib.flow.Flow.from_client_secrets_file(
      CLIENT_SECRETS_FILE, scopes=SCOPES, state=state)
    flow.redirect_uri = REDIRECT_URI

    authorization_response = request.build_absolute_uri()
    #print('authorization response')
    #pprint.pp(authorization_response)

    temp_var = request.build_absolute_uri()
    if "http:" in temp_var:
        temp_var = "https:" + temp_var[5:]

    flow.fetch_token(authorization_response=temp_var)

    # Store credentials in the session.
    # ACTION ITEM: In a production app, you likely want to save these
    # credentials in a persistent database instead.
    credentials = flow.credentials
    request.session['credentials'] = credentials_to_dict(credentials)
    return redirect('closet')

def home(request):
    return render(request, '../templates/home_page.html', {})

def get_started(request):
    return render(request, '../templates/get_started.html', {})

def closet_view(request):
    closet_items = ClosetItem.objects.all()
    for item in closet_items:
        print(item.image)
    dict = {
        'closet_items': closet_items
    }
    return render(request, '../templates/closet.html', dict)

def add_closet_view(request):
    return render(request, '../templates/add_to_closet.html', {})

# def get_started(request):
#     return render(request, 'read_email/get_started.html', {})

# def login(request):
#     return render(request, 'read_email/login.html', {})


from numpy import *

lsstr = ["Abhijeet","Divya","Sujay"]
lsint = [56,16,69]

arrstr = array(lsstr)
arrint = array(lsint)

#array module takes 2 parameter as datatype
# and it is not able to understand what kind of data is arrint
arrmix = array(arrstr,arrint)
print(arrmix)


import os
import prody as prd
import math
from collections import defaultdict
import itertools
import pandas as pd
from multiprocessing import Pool

def distance(distances_1, distances_2):
    x1, y1, z1 = distances_1
    x2, y2, z2 = distances_2
    distance = math.sqrt(math.pow(x2 - x1, 2) +
               math.pow(y2 - y1, 2) +
               math.pow(z2 - z1, 2) * 1.0)
    return distance

def ss_bonds(pdb):

    try:
        cysteines = defaultdict(list)
        free_cys = 0
        cys_bridges = 0

        prot = prd.parsePDB(pdb, QUIET=True)
        for residue in prot.iterResidues():
            name_id = str(residue).split()
            name_id. insert(0, str(residue.getChain()).split()[1])
            if name_id[1] == 'CYS':
                cysteines['_'.join(name_id)].extend(list(residue.getAtom('SG').getCoords()))
                #print('_'.join(name_id))

        ss_bonds = []

        for pair in itertools.combinations(cysteines.keys(), r=2):
            if pair[0] != pair[1]:
                d = distance(cysteines[pair[0]], cysteines[pair[1]])
                if d <= 4.0:
                    ss_bonds.append((pair) + (d, ))
                    cys_bridges += 1
                else:
                    free_cys +=1

    except Exception:
        free_cys, cys_bridges = 0, 0

    return free_cys, cys_bridges

cpus = 70

path = '/storage/evagsm/nobackup/sensitivity_h/'
files = [path + f for f in os.listdir(path) if f.endswith('.pdb')]

pool = Pool(processes = cpus)
results = pool.map(ss_bonds, files)
pool.close()
pool.join()

print(len(results))

df = pd.DataFrame(results)
df.columns = ['free_cys', 'cys_bridges']
df.insert(0, 'ID', [s.split('/')[-1][:-4] for s in files])
df.to_csv('./sens_CYS.csv', index=None)


#напишіть сервер який би отримував у користувача фразу
#а потім надсилав би підраховану кількість слів у відповідь

import socket

abon = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
abon.connect(("localhost", 61487))

while True:
    data = abon.recv(1024)
    print(data.decode("utf-8"))
    abon.send(input(" input text: ").encode("utf-8"))


import torch
import torch.nn as nn

class PromptEncoder(torch.nn.Module):
    def __init__(self, template, hidden_size, tokenizer, args):
        super().__init__()

        self.spell_length = sum(template)
        self.hidden_size = hidden_size
        self.tokenizer = tokenizer
        self.args = args
        self.cloze_length = template
        self.cloze_mask = [
            [1] * self.cloze_length[0]
            + [1] * self.cloze_length[1]
            + [1] * self.cloze_length[2]
        ]
        self.cloze_mask = torch.LongTensor(self.cloze_mask).bool().cuda()

        self.seq_indices = torch.LongTensor(list(range(len(self.cloze_mask[0])))).cuda()
        # embedding
        self.embedding = torch.nn.Embedding(len(self.cloze_mask[0]), self.hidden_size).cuda()
        print("init prompt encoder...")

    def forward(self):
        input_embeds = self.embedding(self.seq_indices)
        return input_embeds


from telegram.ext import Updater, CommandHandler, MessageHandler, Filters
import logging

api = '883009969:AAE9PTeYk1lHTZ4U8fGoTXYE586wTAaXlBk'
updater = Updater(token=api, use_context=True)
dispatcher = updater.dispatcher
logging.basicConfig(format='%(asctime)s - %(name)s - %(levelname)s - %(message)s', level=logging.INFO)

dispatcher.bot.send_message(chat_id=-1001404262415, text='OK')

# def start(update, context):
#     context.bot.send_message(chat_id=update.effective_chat.id, text="I'm a bot, please talk to me!")
#
#
# # /start
# start_handler = CommandHandler('start', start)
# dispatcher.add_handler(start_handler)
#
#
# def hello(update, context):
#     update.message.reply_text('Hello {}'.format(update.message.from_user.first_name))
#
#
# updater.dispatcher.add_handler(CommandHandler('hello', hello))
#
#
# def handle_message(bot, update):
#     print("Received", bot.effective_message.text)
#     print(bot.effective_chat.__dict__)
#
#
# handler = MessageHandler(Filters.text | Filters.command, handle_message)
# updater.dispatcher.add_handler(handler)  # ставим обработчик всех текстовы
#
#
# def unknown(update, context):
#     context.bot.send_message(chat_id=update.effective_chat.id, text=" Извините, я не понял эту команду. ")
#
#
# # Этот обработчик должен быть добавлен последним. Если вы добавите его
# # Некоторые сбитые с толку пользователи могут попытаться отправить команды боту, который он не понимает,
# # поэтому вы можете использовать фильтр MessageHandler
# # /unknown
# unknown_handler = MessageHandler(Filters.command, unknown)
# dispatcher.add_handler(unknown_handler)
#
# updater.start_polling()
# updater.idle()


import pygame
from OpenGL.GL import *
from OpenGL.GLU import *
from pygame.locals import *
from OpenGL.GL.shaders import *
import numpy as np
import os
import time
import pyrr

VAO, program, = None, None

def getFileContents(filename):
    p = os.path.join(os.getcwd(), "shaders", filename)
    return open(p, 'r').read()

def rotationMatrix(degree):
    radian = degree * np.pi / 180.0
    mat = np.array([
        [np.cos(radian), -np.sin(radian), 0.0, 0.0],
        [np.sin(radian), np.cos(radian), 0.0, 0.6],
        [0.0, 0.0, 1.0, 0.4],
        [0.0, 0.0, 0.0, 1.0]
    ], dtype=np.float32)
    return mat

def init():
    global VAO, program
    pygame.init()
    display = (500, 500)
    pygame.display.set_mode(display, DOUBLEBUF | OPENGL)
    glClearColor(1.0, 1.0, 1.0, 1.0)
    gluOrtho2D(-10.0, 10.0, -10.0, 10.0)

    vertexShader = compileShader(getFileContents(
        "cube.vertex.shader"), GL_VERTEX_SHADER)
    fragmentShader = compileShader(getFileContents(
        "cube.fragment.shader"), GL_FRAGMENT_SHADER)

    program = glCreateProgram()
    glAttachShader(program, vertexShader)
    glAttachShader(program, fragmentShader)
    glLinkProgram(program)

    # 36 vertices and 12 triangles to draw cube
    vertexes = np.array([
        # bottom face
        -0.5, -0.5, -0.5, 1.0, 0.0,0.0,
        0.5, -0.5, -0.5, 1.0, 0.0,0.0,
        0.5, 0.5, -0.5, 1.0, 0.0,0.0,

        0.5, 0.5, -0.5, 1.0, 0.0,0.0,
        -0.5, 0.5, -0.5, 1.0, 0.0,0.0,
        -0.5, -0.5, -0.5, 1.0, 0.0,0.0,
        # top face
        -0.5, -0.5, 0.5, 1.0, 1.0,0.0,
        0.5, -0.5, 0.5, 1.0, 1.0,0.0,
        0.5, 0.5, 0.5, 1.0, 1.0,0.0,

        0.5, 0.5, 0.5, 1.0, 1.0,0.0,
        -0.5, 0.5, 0.5, 1.0, 1.0,0.0,
        -0.5, -0.5, 0.5, 1.0, 1.0,0.0,
        # back face
        -0.5, 0.5, 0.5, .0, 1.0,0.0,
        -0.5, 0.5, -0.5, .0, 1.0,0.0,
        -0.5, -0.5, -0.5, .0, 1.0,0.0,

        -0.5, -0.5, -0.5, .0, 1.0,0.0,
        -0.5, -0.5, 0.5, .0, 1.0,0.0,
        -0.5, 0.5, 0.5, .0, 1.0,0.0,
        # front face
        0.5, 0.5, 0.5, 0.0, 0.0,0.0,
        0.5, 0.5, -0.5, 0.0, 0.0,0.0,
        0.5, -0.5, -0.5, 0.0, 0.0,0.0,

        0.5, -0.5, -0.5, 0.0, 0.0,0.0,
        0.5, -0.5, 0.5, 0.0, 0.0,0.0,
        0.5, 0.5, 0.5, 0.0, 0.0,0.0,
        # left face
        -0.5, -0.5, -0.5, 0.0, 0.0,1.0,
        0.5, -0.5, -0.5,  0.0, 0.0,1.0,
        0.5, -0.5, 0.5,  0.0, 0.0,1.0,

        0.5, -0.5, 0.5,  0.0, 0.0,1.0,
        -0.5, -0.5, 0.5,  0.0, 0.0,1.0,
        -0.5, -0.5, -0.5,  0.0, 0.0,1.0,
        # right face
        -0.5, 0.5, -0.5,  1.0, 0.0,1.0,
        0.5, 0.5, -0.5,  1.0, 0.0,1.0,
        0.5, 0.5, 0.5,  1.0, 0.0,1.0,

        0.5, 0.5, 0.5, 1.0, 0.0, 1.0,
        -0.5, 0.5, 0.5, 1.0, 0.0, 1.0,
        -0.5, 0.5, -0.5, 1.0, 0.0, 1.0,

    ], dtype=np.float32)

    VAO = glGenVertexArrays(1)
    VBO = glGenBuffers(1)

    glBindBuffer(GL_ARRAY_BUFFER, VBO)
    glBufferData(GL_ARRAY_BUFFER, vertexes.nbytes, vertexes, GL_STATIC_DRAW)
    glBindVertexArray(VAO)

    position= glGetAttribLocation(program, "position")
    glVertexAttribPointer(position, 3, GL_FLOAT, GL_FALSE,
                          6 * vertexes.itemsize, ctypes.c_void_p(0))
    glEnableVertexAttribArray(position)

    color = glGetAttribLocation(program, "color")
    glVertexAttribPointer(color, 3, GL_FLOAT, GL_FALSE,
                          6* vertexes.itemsize, ctypes.c_void_p(12))
    glEnableVertexAttribArray(color)

def draw():
    global VAO, program
    # enable clearing depth buffer
    glClear(GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT)
    glUseProgram(program)
    # enable depth test
    glEnable(GL_DEPTH_TEST)
    # apply depth checking function
    glDepthFunc(GL_LESS)
    glBindVertexArray(VAO)

    rotation = glGetUniformLocation(program, "transform")
    # rotation = rotationMatrix(30)
    rot_x =rotationMatrix(10 * time.time()) # rotation along axis (0, 0.6, 0.4)
    rot_y = rotationMatrix(10 * time.time()) # rotation along axis (0, 0.6, 0.4)
    glUniformMatrix4fv(rotation, 1, GL_FALSE, pyrr.matrix44.multiply( rot_x,rot_y))
    glDrawArrays(GL_TRIANGLES, 0, 36)

def main():
    init()
    while True:
        for event in pygame.event.get():
            if event.type == pygame.QUIT:
                pygame.quit()
                quit()
        draw()
        pygame.display.flip()
        pygame.time.wait(10)

main()


from view.view import menuList, todoDisplay, inputTodo, updateContent, deleteContent
from entity.todo import Todo
from controller.controller import TodoController

controller = TodoController()

controller.file_read()

while True:
    controller.refresh()
    menu = menuList()
    if menu==0 :
        print("시스템을 종료합니다")
        controller.file_write()
        break
    elif menu==1 :
        todo = inputTodo()
        controller.register(todo)
    elif menu==2:
        todoDisplay(controller.getAllTodos())
    elif menu==3:
        index, content = updateContent()
        controller.update(index, content)
    elif menu==4:
        index = deleteContent()
        if index!=-1:
            controller.delete(index)
    else:
        pass


import serial

ser = serial.Serial()
ser.port = "/dev/ttyUSB1"
ser.baudrate = 115200
ser.bytesize = serial.EIGHTBITS
ser.parity = serial.PARITY_NONE
ser.timeout = 1.

ser.open()
print (ser.is_open)

while True:
    line = ser.read()
    print (line)

ser.close()
print (ser.is_open)


import argparse
import itertools
import pandas as pd
import statsmodels.api as sm

import hnelib.runner
import hnelib.pandas

import usfhn.datasets
import usfhn.constants
import usfhn.measurements
import usfhn.views
import usfhn.non_us
import usfhn.fieldwork
import usfhn.attrition
import usfhn.careers
import usfhn.interdisciplinarity
import usfhn.new_hires
import usfhn.self_hiring
import usfhn.taxonomy
import usfhn.rank
import usfhn.gender
import usfhn.pool_reduction
import usfhn.demographics
import usfhn.closedness
import usfhn.webs
import usfhn.non_doctorate
import usfhn.stats_utils

################################################################################
#
#
# Utilities for standard names
#
#
################################################################################
VARIABLE_TO_COLUMN = {
    'ginis': 'GiniCoefficient',
    'gender': 'FractionFemale',
    'self-hires': 'SelfHiresFraction',
    'self-hire': 'SelfHire',
    'non-us': 'NonUS',
    'faculty-rank': 'Rank',
    'seniority': 'Senior',
    'new-hire': 'NewHire',
    'year': 'Year',
    'career-age': 'CareerAge',
}

# higher priority = happens earlier in the path
FLAG_PARAMETER_PRIORITY = {
    'by_year': 0,
}

def is_flag_parameter(parameter):
    """
    so I get in something like `by_faculty_rank`

    return false if doesn't start with `by_`

    replace '_' with '-'

    return true if is in VARIABLE_TO_COLUMN
    """
    if not parameter.startswith('by_'):
        return False

    parameter = parameter.replace('by_', '')
    parameter = parameter.replace('_', '-')

    return parameter in VARIABLE_TO_COLUMN

def get_flag_parameters_from_kwargs(**kwargs):
    """
    we're going to return the flag parameters in a specific order because
    sometimes there are multiple qualifiers
    """
    parameters = [k for k, v in kwargs.items() if v and is_flag_parameter(k)]

    max_priority = max(FLAG_PARAMETER_PRIORITY.values()) + 1
    return sorted(parameters, key=lambda p: FLAG_PARAMETER_PRIORITY.get(p, max_priority), reverse=True)

def get_variable_from_flag_parameter(string):
    string = string.replace('by_', '')
    string = string.replace('_', '-')
    return string

def is_variable_qualifier(string):
    if not string.startswith('by-'):
        return False

    string = string.replace('by-', '')

    return string in VARIABLE_TO_COLUMN

def get_groupby_cols_from_kwargs(exclude=[], **kwargs):
    parameters = get_flag_parameters_from_kwargs(**kwargs)
    variables = [get_variable_from_flag_parameter(p) for p in parameters]
    groupby_cols = [VARIABLE_TO_COLUMN[v] for v in variables]
    groupby_cols = [c for c in groupby_cols if c not in exclude]
    return groupby_cols

def get_variable_dataframe(variable, suffix=None, item_kwargs={}, **kwargs):
    flag_parameters = get_flag_parameters_from_kwargs(**kwargs)
    qualifiers = [p.replace('_', '-') for p in flag_parameters]

    parts = [variable] + qualifiers + [suffix]
    path = "/".join([p for p in parts if p])
    df = runner.get(path, **item_kwargs)

    return df

def add_groupby_annotations_to_df(
    df,
    by_gender=False,
    by_new_hire=False,
    by_non_us=False,
    by_faculty_rank=False,
    by_seniority=False,
    by_career_age=False,
    by_self_hire=False,
    explode_gender=False,
    **kwargs,
):
    cols = list(df.columns)

    groupby_cols = []

    if by_gender:
        df = usfhn.gender.annotate_gender(df, explode_gender=explode_gender)
        groupby_cols.append('Gender')

    if by_career_age:
        df = usfhn.careers.annotate_career_age(df)
        groupby_cols.append('CareerAge')

    if by_faculty_rank:
        df = usfhn.rank.annotate_faculty_rank(df)
        groupby_cols.append('Rank')

    if by_seniority:
        df = usfhn.rank.annotate_seniority(df)
        groupby_cols.append('Senior')

    if by_non_us:
        df = usfhn.non_us.annotate_non_us(df)
        groupby_cols.append('NonUS')

    if by_new_hire:
        df = usfhn.new_hires.annotate_new_hires(df)
        groupby_cols.append('NewHire')

    if by_self_hire:
        df = usfhn.self_hiring.annotate_self_hires(df)
        groupby_cols.append('SelfHire')

    df = df[
        cols + groupby_cols
    ].drop_duplicates()

    return df, groupby_cols

################################################################################
#
#
# Expanders and stats collections
#
#
################################################################################
RANKS_EXPANDER = {'rank_type': ['production', 'prestige']}
ALL_RANKS_EXPANDER = {'rank_type': ['production', 'doctoral-institution', 'employing-institution']}

def get_attrition_risk_stats_functions():
    return {
        'taxonomy': usfhn.attrition.get_taxonomy_attrition_risk,
        'gender': usfhn.attrition.get_gender_attrition_risk,
        'self-hires': usfhn.attrition.get_self_hire_attrition_risk,
        'institution': {
            'do': usfhn.attrition.get_institution_attrition_risk,
            'prefix_expansions': {'institution_column': ['DegreeInstitutionId', 'InstitutionId']},
        },
        'us': usfhn.attrition.get_non_us_attrition_risk,
        'non-us-by-is-english': usfhn.attrition.get_non_us_by_is_english_attrition_risk,
        'rank-change': {
            'do': usfhn.attrition.get_rank_change_attrition_risk,
            'directory_expansions': RANKS_EXPANDER,
        },
    }

def get_rank_stats_functions():
    return {
        'directory_expansions': RANKS_EXPANDER,
        'df': usfhn.rank.get_ranks_df_for_stats,
        'change': usfhn.rank.get_rank_change_df,
        'hierarchy-stats': usfhn.rank.get_hierarchy_stats,
        'mean-change': usfhn.rank.get_mean_rank_change,
        'movement-distance': usfhn.rank.get_gendered_movement_distance_significance,
        'gender': {
            'kwargs': {'by_gender': True},
            'hierarchy-stats': usfhn.rank.get_hierarchy_stats,
            'mean-change': usfhn.rank.get_mean_rank_change,
            'placements': usfhn.rank.get_placements,
        },
        'seniority': {
            'kwargs': {'by_seniority': True},
            'hierarchy-stats': usfhn.rank.get_hierarchy_stats,
            'mean-change': usfhn.rank.get_mean_rank_change,
        },
    }

################################################################################
#
#
# basics
#
#
################################################################################
def get_faculty_hiring_network(by_year=False, use_closedness_data=False, by_gender=False):
    """
    Columns:
    - DegreeInstitutionId: producing institution id
    - InstitutionId: employing institution id
    - Count: faculty at InstitutionId with PhDs from DegreeInstitutionId
    - OutDegree: total faculty with PhDs from DegreeInstitutionId
    - InDegree: total faculty employed by InstitutionId
    - TotalCount: total faculty

    we include zeroes in when the level is academia because we know that these
    are phd-granting institutions, so they are producers.

    we do not know whether a particular phd-granting institution is phd granting
    in any particular field/area/umbrella, however, so we exclude them to arrive
    at a lower bound.
    """
    groupby_cols = [
        'TaxonomyLevel',
        'TaxonomyValue',
    ]

    df_name = 'closedness_data' if use_closedness_data else 'data'

    if by_year:
        groupby_cols.append('Year')

    df = usfhn.datasets.get_dataset_df(df_name, by_year=by_year)

    df = usfhn.fieldwork.linearize_df_taxonomy_hierarchy(df)

    df = df[
        groupby_cols + [
            'InstitutionId',
            'DegreeInstitutionId',
            'PersonId',
        ]
    ].drop_duplicates()

    df, extra_groupby_cols = add_groupby_annotations_to_df(df, by_gender=by_gender)
    groupby_cols += extra_groupby_cols

    df['Count'] = df.groupby(
        ['InstitutionId', 'DegreeInstitutionId'] + groupby_cols
    )['PersonId'].transform('nunique').fillna(0)

    df['OutDegree'] = df.groupby(
        ['DegreeInstitutionId'] + groupby_cols
    )['PersonId'].transform('nunique').fillna(0)

    df['InDegree'] = df.groupby(
        ['InstitutionId'] + groupby_cols
    )['PersonId'].transform('nunique').fillna(0)

    df['TotalCount'] = df.groupby(
        groupby_cols
    )['PersonId'].transform('nunique').fillna(0)

    df = df.drop(columns=['PersonId']).drop_duplicates()

    df = df[
        (df['Count'] > 0)
        |
        (df['TaxonomyLevel'] == 'Academia')
    ]

    return df

def get_faculty_production(by_year=False, new_hires=False, non_new_hires=False, non_us=False):
    """
    We include zeroes in when the level is academia because we KNOW that these
    are PhD-granting institutions, so they are Producers.

    We do not know whether a particular PhD-granting institution is PhD granting
    in any particular field/area/umbrella, however, so we exclude them to arrive
    at a lower bound.
    """
    groupby_cols = [
        'TaxonomyLevel',
        'TaxonomyValue',
    ]

    dataset_name = 'closedness_data' if non_us else 'data'

    if by_year:
        groupby_cols.append('Year')

    df = usfhn.datasets.get_dataset_df(dataset_name, by_year=by_year)
    df = usfhn.fieldwork.linearize_df_taxonomy_hierarchy(df)

    df = df[
        groupby_cols + [
            'PersonId',
            'DegreeInstitutionId',
            'InstitutionId',
        ]
    ].drop_duplicates()

    if new_hires or non_new_hires:
        if by_year:
            new_hires_df = runner.get('new-hire/by-year/df')
        else:
            new_hires_df = runner.get('new-hire/df')

        new_hires_df['NewHire'] = True

        df = df.merge(
            new_hires_df,
            on=list(df.columns),
            how='left',
        )

        df['NewHire'] = df['NewHire'].fillna(False)

        if non_new_hires:
            df = df[~df['NewHire']]
        else:
            df = df[df['NewHire']]

        df = df.drop(columns=['NewHire'])

    df['ProductionCount'] = df.groupby(
        groupby_cols + ['DegreeInstitutionId']
    )['PersonId'].transform('nunique').fillna(0)

    df = df[
        (df['ProductionCount'] > 0)
        |
        (df['TaxonomyLevel'] == 'Academia')
    ]

    df['TotalProduction'] = df.groupby(groupby_cols)['PersonId'].transform('nunique')

    df = df.drop(columns=['PersonId']).drop_duplicates()

    df['ProductionFraction'] = df['ProductionCount'] / df['TotalProduction']
    df['ProductionPercent'] = 100 * df['ProductionFraction']

    return df

def get_faculty_hiring(by_year=False):
    """
    We include zeroes in when the level is academia because we KNOW that these
    are PhD-granting institutions, so they are Producers.

    We do not know whether a particular PhD-granting institution is PhD granting
    in any particular field/area/umbrella, however, so we exclude them to arrive
    at a lower bound.
    """
    groupby_cols = [
        'TaxonomyLevel',
        'TaxonomyValue',
    ]

    if by_year:
        groupby_cols.append('Year')

    df = usfhn.datasets.get_dataset_df('data', by_year=by_year)

    df = usfhn.fieldwork.linearize_df_taxonomy_hierarchy(df)

    df = df[
        groupby_cols + [
            'DegreeInstitutionId',
            'InstitutionId',
            'PersonId',
        ]
    ].drop_duplicates()

    df['HiredFaculty'] = df.groupby(
        groupby_cols + ['InstitutionId', 'DegreeInstitutionId',]
    )['PersonId'].transform('nunique').fillna(0)

    df['TotalHiredFaculty'] = df.groupby(groupby_cols)['PersonId'].transform('nunique')

    df = df.drop(columns=['PersonId']).drop_duplicates()

    df = df[
        (df['HiredFaculty'] > 0)
        |
        (df['TaxonomyLevel'] == 'Academia')
    ]

    df['HiredFacultyFraction'] = df['HiredFaculty'] / df['TotalHiredFaculty']

    return df

def get_lorenz_curves(by_year=False, new_hires=False, non_new_hires=False):
    groupby_cols = [
        'TaxonomyLevel',
        'TaxonomyValue',
    ]

    if by_year:
        groupby_cols.append('Year')

        if new_hires:
            df = runner.get('new-hire/by-year/production')
        else:
            df = runner.get('basics/by-year/production')
    else:
        if new_hires:
            df = runner.get('new-hire/production')
        elif non_new_hires:
            df = runner.get('non-new-hire/production')
        else:
            df = runner.get('basics/production')

    curves = []
    for _, rows in df.groupby(groupby_cols):
        rows = rows.drop_duplicates(subset=['DegreeInstitutionId'])
        curve = usfhn.measurements.lorenz_curve(rows['ProductionFraction'])
        for col in groupby_cols:
            curve[col] = rows.iloc[0][col]

        curves.append(curve)

    return pd.concat(curves)

################################################################################
#
#
# Changes over time
#
#
################################################################################
def run_ols_and_get_slopes(variable, p_threshold=.05, **kwargs):
    df = get_variable_dataframe(variable, suffix='df', by_year=True, **kwargs)

    groupby_cols = ['TaxonomyLevel', 'TaxonomyValue']
    groupby_cols += get_groupby_cols_from_kwargs(**kwargs)

    value_col = VARIABLE_TO_COLUMN[variable]

    result_rows = []
    for _, rows in df.groupby(groupby_cols):
        groupby_info = {c: rows.iloc[0][c] for c in groupby_cols}

        X = sm.add_constant(rows['Year'] - (min(rows['Year']) - 1))
        Y = rows[value_col]
        model = sm.OLS(Y, X)
        result = model.fit()

        result_rows.append({
            **groupby_info,
            'Slope': result.params['Year'],
            'R^2': result.rsquared,
            'P': result.pvalues['Year'],
            'Significant': p_threshold > result.pvalues['Year'],
        })

    df = pd.DataFrame(result_rows)

    df = usfhn.stats_utils.correct_multiple_hypotheses(df)
    df['Significant'] = df['PCorrected'] < p_threshold

    return pd.DataFrame(result_rows)

def degree_years_by_taxonomy():
    df = usfhn.datasets.CURRENT_DATASET.closedness_data[
        [
            'PersonId',
            'DegreeYear',
            'Taxonomy',
        ]
    ].drop_duplicates()

    df = usfhn.fieldwork.linearize_df_taxonomy_hierarchy(df)

    return df

def get_years_until_like_new_hires(variable, significant=True):
    value_col = VARIABLE_TO_COLUMN[variable]

    df = get_variable_dataframe(variable, suffix='df', by_new_hire=True)
    df = hnelib.pandas.aggregate_df_over_column(
        df,
        agg_col='NewHire',
        join_cols=['TaxonomyLevel', 'TaxonomyValue'],
        value_cols=[value_col],
        agg_value_to_label={
            True: 'NewHire',
            False: 'ExistingHire',
        },
    )

    df['Difference'] = df[f'NewHire{value_col}'] - df[f'ExistingHire{value_col}']
    df = df[
        [
            'TaxonomyLevel',
            'TaxonomyValue',
            'Difference',
        ]
    ].drop_duplicates()

    ols_df = get_variable_dataframe(variable, suffix='slopes')

    if significant:
        ols_df = ols_df[
            ols_df['Significant']
        ]

    ols_df = ols_df[
        [
            'TaxonomyLevel',
            'TaxonomyValue',
            'Slope',
            'Significant',
            'P',
        ]
    ].drop_duplicates()

    df = df.merge(
        ols_df,
        on=[
            'TaxonomyLevel',
            'TaxonomyValue',
        ]
    )

    df['YearsToProduceChange'] = df['Difference'] / df['Slope']

    df = df[
        [
            'TaxonomyLevel',
            'TaxonomyValue',
            'YearsToProduceChange',
        ]
    ].drop_duplicates()

    df = df[
        df['YearsToProduceChange'] > 0
    ]

    return df

################################################################################
#
#
# Stats collection
#
#
################################################################################
STATS_COLLECTION = {
    # to replace with stats calls:
    # - get_steepness_by_taxonomy
    'basics': {
        'faculty-hiring-network': get_faculty_hiring_network,
        'closedness-faculty-hiring-network': {
            'kwargs': {'use_closedness_data': True},
            'df': get_faculty_hiring_network,
            'by-gender': {
                'kwargs': {'by_gender': True},
                'do': get_faculty_hiring_network,
            },
        },
        'production': get_faculty_production,
        'hiring': get_faculty_hiring,
        'lorenz': get_lorenz_curves,
    },
    'demographics': {
        'degree-years': degree_years_by_taxonomy,
        'non-doctorates-by-taxonomy': usfhn.demographics.get_non_doctorate_fraction_by_taxonomy,
    },
    'closedness': {
        'df': usfhn.closedness._get_closednesses,
    },
    'taxonomy': {
        'df': usfhn.taxonomy.get_taxonomy_size,
        'institutions': {
            'kwargs': {'count_col': 'InstitutionId'},
            'do': usfhn.taxonomy.get_taxonomy_size,
        },
        'by-year': {
            'kwargs': {'by_year': True},
            'df': usfhn.taxonomy.get_taxonomy_size,
            'institutions': {
                'kwargs': {'count_col': 'InstitutionId'},
                'do': usfhn.taxonomy.get_taxonomy_size,
            },
        },
    },
    'ginis': {
        'kwargs': {'variable': 'ginis'},
        'df': usfhn.gini.get_gini_coefficients,
        'slopes': run_ols_and_get_slopes,
        'stasis': get_years_until_like_new_hires,
        'by-year': {
            'kwargs': {'by_year': True},
            'df': usfhn.gini.get_gini_coefficients,
        },
        'by-faculty-rank': {
            'kwargs': {'by_faculty_rank': True},
            'df': usfhn.gini.get_gini_coefficients,
            'slopes': run_ols_and_get_slopes,
            'by-year': {
                'kwargs': {'by_year': True},
                'df': usfhn.gini.get_gini_coefficients,
            },
        },
        'by-seniority': {
            'kwargs': {'by_seniority': True},
            'df': usfhn.gini.get_gini_coefficients,
            'slopes': run_ols_and_get_slopes,
            'by-year': {
                'kwargs': {'by_year': True},
                'df': usfhn.gini.get_gini_coefficients,
            },
        },
        'by-new-hire': {
            'kwargs': {'by_new_hire': True},
            'df': usfhn.gini.get_gini_coefficients,
            'slopes': run_ols_and_get_slopes,
            'by-year': {
                'kwargs': {'by_year': True},
                'df': usfhn.gini.get_gini_coefficients,
            },
        },
    },
    'gender': {
        'kwargs': {'variable': 'gender'},
        'df': usfhn.gender.get_taxonomy_gender_fraction,
        'slopes': run_ols_and_get_slopes,
        'stasis': get_years_until_like_new_hires,
        'by-year': {
            'kwargs': {'by_year': True},
            'df': usfhn.gender.get_taxonomy_gender_fraction,
        },
        'by-faculty-rank': {
            'kwargs': {'by_faculty_rank': True},
            'df': usfhn.gender.get_taxonomy_gender_fraction,
            'slopes': run_ols_and_get_slopes,
            'by-year': {
                'kwargs': {'by_year': True},
                'df': usfhn.gender.get_taxonomy_gender_fraction,
            },
        },
        'by-seniority': {
            'kwargs': {'by_seniority': True},
            'df': usfhn.gender.get_taxonomy_gender_fraction,
            'slopes': run_ols_and_get_slopes,
            'by-year': {
                'kwargs': {'by_year': True},
                'df': usfhn.gender.get_taxonomy_gender_fraction,
            },
        },
        'by-new-hire': {
            'kwargs': {'by_new_hire': True},
            'df': usfhn.gender.get_taxonomy_gender_fraction,
            'slopes': run_ols_and_get_slopes,
            'by-year': {
                'kwargs': {'by_year': True},
                'df': usfhn.gender.get_taxonomy_gender_fraction,
            },
            'by-rank': {
                'directory_expansions': RANKS_EXPANDER,
                'logits': usfhn.gender.rank_vs_gender_logit,
                'existing-hire-logits': {
                    'kwargs': {'existing_hires': True},
                    'do': usfhn.gender.rank_vs_gender_logit,
                },
            },
        },
        'by-non-us': {
            'kwargs': {'by_non_us': True},
            'df': usfhn.gender.get_taxonomy_gender_fraction,
        },
        'by-career-age': {
            'kwargs': {'by_career_age': True},
            'df': usfhn.gender.get_taxonomy_gender_fraction,
        },
        'by-rank': {
            'directory_expansions': RANKS_EXPANDER,
            'logits': usfhn.gender.rank_vs_gender_logit,
        },
    },
    'self-hires': {
        'kwargs': {'variable': 'self-hires'},
        'df': usfhn.self_hiring.get_taxonomy_self_hiring,
        'slopes': run_ols_and_get_slopes,
        'stasis': get_years_until_like_new_hires,
        'by-year': {
            'kwargs': {'by_year': True},
            'df': usfhn.self_hiring.get_taxonomy_self_hiring,
        },
        'by-faculty-rank': {
            'kwargs': {'by_faculty_rank': True},
            'df': usfhn.self_hiring.get_taxonomy_self_hiring,
            'slopes': run_ols_and_get_slopes,
            'by-year': {
                'kwargs': {'by_year': True},
                'df': usfhn.self_hiring.get_taxonomy_self_hiring,
            },
        },
        'by-gender': {
            'kwargs': {'by_gender': True},
            'df': usfhn.self_hiring.get_taxonomy_self_hiring,
        },
        'by-seniority': {
            'kwargs': {'by_seniority': True},
            'df': usfhn.self_hiring.get_taxonomy_self_hiring,
            'slopes': run_ols_and_get_slopes,
            'by-year': {
                'kwargs': {'by_year': True},
                'df': usfhn.self_hiring.get_taxonomy_self_hiring,
            },
        },
        'by-new-hire': {
            'kwargs': {'by_new_hire': True},
            'df': usfhn.self_hiring.get_taxonomy_self_hiring,
            'slopes': run_ols_and_get_slopes,
            'by-year': {
                'kwargs': {'by_year': True},
                'df': usfhn.self_hiring.get_taxonomy_self_hiring,
            },
            'by-rank': {
                'directory_expansions': RANKS_EXPANDER,
                'logits': usfhn.self_hiring.rank_vs_self_hires_logit,
                'existing-hire-logits': {
                    'kwargs': {'existing_hires': True},
                    'do': usfhn.self_hiring.rank_vs_self_hires_logit,
                },
            },
        },
        'by-career-age': {
            'kwargs': {'by_career_age': True},
            'df': usfhn.self_hiring.get_taxonomy_self_hiring,
        },
        'by-institution': {
            'kwargs': {'by_institution': True},
            'df': usfhn.self_hiring.get_taxonomy_self_hiring,
        },
        'by-rank': {
            'logits': {
                'directory_expansions': RANKS_EXPANDER,
                'do': usfhn.self_hiring.rank_vs_self_hires_logit,
            },
            'binned-logits': {
                'directory_expansions': ALL_RANKS_EXPANDER,
                'do': usfhn.self_hiring.self_hiring_by_career_age_logistics,
            },
        },
    },
    'non-us': {
        'df': usfhn.non_us.get_fraction_non_us,
        'by-year': {
            'kwargs': {'by_year': True},
            'df': usfhn.non_us.get_fraction_non_us,
        },
        'by-institution': {
            'df': usfhn.non_us.get_fraction_non_us_by_institution,
            'by-rank': {
                'directory_expansions': RANKS_EXPANDER,
                'df': usfhn.non_us.get_fraction_non_us_by_institution_with_rank,
                'logits': usfhn.non_us.rank_vs_non_us_logit,
            },
        },
        'by-new-hire': {
            'kwargs': {'by_new_hire': True},
            'by-rank': {
                'directory_expansions': RANKS_EXPANDER,
                'logits': usfhn.non_us.rank_vs_non_us_logit,
                'existing-hire-logits': {
                    'kwargs': {'existing_hires': True},
                    'do': usfhn.non_us.rank_vs_non_us_logit,
                },
            },
        },
        'by-career-age': {
            'kwargs': {'by_career_age': True},
            'df': usfhn.non_us.get_fraction_non_us,
            'by-continent': {
                'kwargs': {'by_continent': True},
                'df': usfhn.non_us.get_fraction_non_us_by_continent_and_career_age,
            },
        },
        'by-country': usfhn.non_us.non_us_countries_by_production,
        'by-continent': usfhn.non_us.non_us_continents_by_production,
        'production': {
            'kwargs': {'non_us': True},
            'do': get_faculty_production,
        },
    },
    'ranks': {
        # we run a lot of the same stats for ranks in different things, so put
        # them into a function. functions:
        # - df
        # - change
        # - hierarchy-stats
        # - mean-change
        **get_rank_stats_functions(),
        'institution-rank-correlations': {
            'directory_expansions': RANKS_EXPANDER,
            'do': usfhn.rank.get_field_to_field_rank_correlations,
        },
        'by-year': {
            'kwargs': {'by_year': True},
            **get_rank_stats_functions(),
        },
        'placements': usfhn.rank.get_placements,
        'institution-fields': usfhn.rank.get_institution_fields,
        'institution-placement-stats': usfhn.rank.get_institution_placement_stats,
    },
    'interdisciplinarity': {
        'df': usfhn.interdisciplinarity.get_employment_interdisciplinarity_df,
        'taxonomy': usfhn.interdisciplinarity.get_taxonomy_interdisciplinarity,
        'institution': usfhn.interdisciplinarity.get_institution_interdisciplinarity,
        'degree-institution': usfhn.interdisciplinarity.get_degree_institution_interdisciplinarity,
        'by-year': {
            'kwargs': {'by_year': True},
            'df': usfhn.interdisciplinarity.get_employment_interdisciplinarity_df,
            'taxonomy': usfhn.interdisciplinarity.get_taxonomy_interdisciplinarity,
        },
        'institution-with-ranks': {
            'prefixes': RANKS_EXPANDER,
            'do': usfhn.interdisciplinarity.institutions_with_ranks,
        },
        'by-rank': {
            'directory_expansions': RANKS_EXPANDER,
            'logits': usfhn.interdisciplinarity.rank_vs_interdisciplinarity_logit,
        },
    },
    'careers': {
        'df': usfhn.careers.get_career_moves_df,
        'gender': usfhn.careers.get_move_risk_by_gender, # gender-overall
        'by-year': {
            'gender': usfhn.careers.get_move_risk_by_gender_and_year, # gender-by-year
        },
        'institution-risk': usfhn.careers.get_institution_move_risk,
        'taxonomy-hires-before-and-after-mcms': usfhn.careers.get_taxonomy_hires_before_and_after_mcms,
        'hierarchy-changes-from-mcms': usfhn.careers.get_mcm_stats,
        'hierarchy-changes-from-mcms-compare-to-normal': usfhn.careers.get_mcm_stats_comparison,
        'length': {
            'df': usfhn.careers.get_career_lengths_df,
            'taxonomy': usfhn.careers.career_length_by_taxonomy,
            'gender': usfhn.careers.career_length_by_gender,
            'gini': usfhn.careers.get_gini_coefficients_by_career_length,
            'by-year': {
                'kwargs': {'by_year': True},
                'df': usfhn.careers.get_career_lengths_df,
                'gender': usfhn.careers.career_length_by_gender,
                'taxonomy': usfhn.careers.career_length_by_taxonomy,
                'gini': usfhn.careers.get_gini_coefficients_by_career_length,
            },
            'by-rank': {
                'directory_expansions': RANKS_EXPANDER,
                'logits': usfhn.careers.rank_vs_career_length_logit,
            },
        },
    },
    'new-hire': {
        'df': usfhn.new_hires.get_new_hires_df,
        'self-hires': usfhn.new_hires.get_self_hires,
        'non-us': usfhn.non_us.get_new_hire_fraction,
        'interdisciplinarity': usfhn.interdisciplinarity.get_new_hire_fraction,
        'steepness': usfhn.new_hires.get_steepness,
        'production': {
            'kwargs': {'new_hires': True},
            'do': get_faculty_production,
        },
        'lorenz': {
            'kwargs': {'new_hires': True},
            'do': get_lorenz_curves,
        },
        'by-year': {
            'kwargs': {'by_year': True},
            'df': usfhn.new_hires.get_new_hires_df,
            'self-hires': usfhn.new_hires.get_self_hires,
            'non-us': usfhn.non_us.get_new_hire_fraction,
            'interdisciplinarity': usfhn.interdisciplinarity.get_new_hire_fraction,
            'steepness': usfhn.new_hires.get_steepness,
        },
    },
    'non-new-hire': {
        'production': {
            'kwargs': {'non_new_hires': True},
            'do': get_faculty_production,
        },
        'lorenz': {
            'kwargs': {'non_new_hires': True},
            'do': get_lorenz_curves,
        },
    },
    'non-doctorate': {
        'df': usfhn.non_doctorate.get_non_doctorates_by_taxonomy,
        'by-year': {
            'kwargs': {'by_year': True},
            'df': usfhn.non_doctorate.get_non_doctorates_by_taxonomy,
        },
    },
    'attrition': {
        'df': usfhn.attrition.get_attritions_df,
        'by-gender': {
            'age-of-attrition': usfhn.attrition.mean_age_of_attrition,
        },
        'by-self-hire': {
            'risk-ratio': usfhn.attrition.self_hire_to_non_self_hire_risk_ratio,
            'prestige-deciles': {
                'directory_expansions': ALL_RANKS_EXPANDER,
                'do': usfhn.self_hiring.self_hiring_binned_by_prestige_attrition_risk,
            },
        },
        'risk': {
            # for risk, we run a LOT of different risk combinations, so I put
            # them in a function. The keys are:
            # - taxonomy
            # - gender
            # - self-hires
            # - institution, which is expanded over:
            #   - institution_column: ['DegreeInstitutionId', 'InstitutionId']
            **get_attrition_risk_stats_functions(),
            'by-career-stage': {
                'kwargs': {'by_career_stage': True},
                **get_attrition_risk_stats_functions(),
            },
            'by-time': {
                'kwargs': {'by_year': True},
                **get_attrition_risk_stats_functions(),
                'by-career-stage': {
                    'kwargs': {'by_career_stage': True},
                    **get_attrition_risk_stats_functions(),
                },
                'by-degree-year': {
                    'kwargs': {'by_degree_year': True},
                    **get_attrition_risk_stats_functions(),
                },
            },
            'by-degree-year': {
                'kwargs': {'by_degree_year': True},
                **get_attrition_risk_stats_functions(),
            },
            'gender-balanced-hiring': usfhn.attrition.get_gender_balance_given_attrition_and_balanced_hiring,
        },
        'by-rank': {
            'directory_expansions': ALL_RANKS_EXPANDER,
            'df': usfhn.attrition.attritions_df_with_ranks,
            'institution': {
                'risk': usfhn.attrition.institutions_with_ranks,
                'logits': usfhn.attrition.rank_vs_attrition_risk_logit,
                'gendered-logits': {
                    'kwargs': {'by_gender': True},
                    'do': usfhn.attrition.rank_vs_attrition_risk_logit,
                },
                'by-self-hire': {
                    'kwargs': {'by_self_hire': True},
                    'logits': usfhn.attrition.rank_vs_attrition_risk_logit,
                },
            },
            'academia-ranks': {
                'kwargs': {'academia_rank_only': True},
                'df': usfhn.attrition.attritions_df_with_ranks,
                'institution': {
                    'logits': usfhn.attrition.rank_vs_attrition_risk_logit,
                },
            }
        },
    },
    'pool-reduction': {
        'umbrella-stats': usfhn.pool_reduction.umbrella_exclusion_stats,
        'excluded-taxonomy-sizes': usfhn.pool_reduction.excluded_taxonomy_sizes,
        'excluded-at-field-level': {
            'do': usfhn.pool_reduction.people_excluded_at_field_level_by_umbrella,
            'prefix_expansions': {'dataset': ['unfiltered-census-pool', 'alternate-fields']},
        },
        'excluded-at-umbrella-level': {
            'do': usfhn.pool_reduction.people_excluded_at_domain_level_by_umbrella,
            'prefix_expansions': {'dataset': ['unfiltered-census-pool', 'alternate-fields']},
        },
        'paper-stats': {
            'expansion_type': hnelib.runner.JSONExpansion,
            'general': usfhn.pool_reduction.get_stats,
        },
    },
    'geodesics': {
        'df': usfhn.webs.get_geodesics_df,
    },
}

runner = hnelib.runner.DataFrameRunner(
    collection=STATS_COLLECTION,
    directory=usfhn.constants.STATS_PATH,
)

if __name__ == '__main__':
    parser = argparse.ArgumentParser(description='stat runner')
    parser.add_argument('--which', '-w', type=str, help='which thing to run')
    parser.add_argument('--all', default=False, action='store_true')
    parser.add_argument('--run_expansions', '-e', default=False, action='store_true', help='run combinations')
    parser.add_argument('--clean', default=False, action='store_true', help='clean old stats')
    args = parser.parse_args()

    if args.clean:
        runner.clean()

    to_run = []
    if args.which:
        to_run.append(args.which)
    elif args.all:
        to_run = list(runner.collection.keys())

    for item_name in to_run:
        runner.run(item_name, run_expansions=args.run_expansions)


# All functions starting with "test_" should run and execute `print("OK")` exactly once.
# This can be checked by running validTest.py.

import sys
import os

sys.path.append(os.path.dirname(os.path.dirname((__file__))))
from testlib import expects

# These are defined so that we can evaluate the test code.
NONSOURCE = "not a source"
SOURCE = "source"

def is_source(x):
    return x == "source" or x == b"source" or x == 42 or x == 42.0 or x == 42j

def SINK(x):
    if is_source(x):
        print("OK")
    else:
        print("Unexpected flow", x)

def SINK_F(x):
    if is_source(x):
        print("Unexpected flow", x)
    else:
        print("OK")

# ------------------------------------------------------------------------------
# Actual tests
# ------------------------------------------------------------------------------

def test_while():
    x = NONSOURCE
    n = 2
    while n > 0:
        if n == 1:
            SINK(x) #$ flow="SOURCE, l:+1 -> x"
        x = SOURCE
        n -= 1

class MyObj(object):
    def __init__(self, foo):
        self.foo = foo

def setFoo(obj, x):
    obj.foo = x

def test_while_obj():
    myobj = MyObj(NONSOURCE)
    n = 2
    while n > 0:
        if n == 1:
            SINK(myobj.foo) #$ flow="SOURCE, l:+1 -> myobj.foo"
        setFoo(myobj, SOURCE)
        n -= 1

def setAndTestFoo(obj, x, test):
    if test:
        # This flow is not found, if self-loops are broken at the SSA level.
        SINK(obj.foo) #$ flow="SOURCE, l:+7 -> obj.foo"
    obj.foo = x

def test_while_obj_sink():
    myobj = MyObj(NONSOURCE)
    n = 2
    while n > 0:
        setAndTestFoo(myobj, SOURCE, n == 1)
        n -= 1


import enum

class Status(enum.Enum):
    success = 1
    failure = 2

def die(status, message=None, output=[]):
    if message:
        log("die", message, output)

    def print_logs(pretty_log_name, real_names):
        logs = []
        for root, dirs, files in os.walk("/tmp"):
            for f in files:
                for name in real_names:
                    if f == name:
                        logs.append(os.path.join(root, f))
        if logs:
            log("info", "Printing %s" % pretty_log_name)
        for l in logs:
            lines = []
            with open(l, encoding="utf-8", errors="ignore") as f:
                for line in f:
                    lines.append(line.strip())
                log("command", "%s '%s'" % (pretty_log_name, l), lines)

    print_logs("config logfiles", ["config.log", "configure.log"])
    print_logs("Cmake errors", ["CMakeError.log"])
    print_logs("Cmake output", ["CMakeOutput.log"])

    if status == Status.success:
        exit(0)
    elif status == Status.failure:
        exit(1)
    else:
        raise RuntimeError("Bad call to die with '%s'" % str(status))

import argparse
import datetime
import json
import os
import os.path
import random
import re
import shutil
import subprocess
import sys
import textwrap
import tempfile
import time

def run_cmd(cmd, as_root=False, output=True):
    time = timestamp()

    if output:
        cmd_out=subprocess.PIPE
    else:
        cmd_out=subprocess.DEVNULL

    if not as_root:
        cmd = "sudo -u tuscan " + cmd

    cp = subprocess.run(cmd.split(), stdout=cmd_out, stderr=subprocess.STDOUT,
             universal_newlines=True)

    if output:
        lines = cp.stdout.splitlines()
    else:
        lines = []

    if cp.returncode:
        log("die", cmd, lines, time)
        exit(1)
    else:
        log("command", cmd, lines, time)

def interpret_bash_array(pkgbuild, array_name):
    """Return the Bash array array_name in the file at path pkgbuild.

    Returns:
        A list of strings, or an empty string if the array is not
        defined in the PKGBUILD, or None if there was a problem
        interpreting the PKGBUILD.
    """
    cmd = textwrap.dedent("""\
                 #!/bin/bash
                 . {pkgbuild}
                 # This is in case the array is actually an array
                 for foo in ${{{array_name}[@]}}; do
                    echo ${{foo}};
                 done;
                 # This is in case the array is actually a single string
                 echo "${{{array_name}}}"
                 """.format(pkgbuild=pkgbuild, array_name=array_name))
    with tempfile.NamedTemporaryFile(mode="w") as temp:
        temp.write(cmd)
        temp.flush()
        try:
            cp = subprocess.run(["/bin/bash", temp.name], stdout=subprocess.PIPE,
                     universal_newlines=True, timeout=20)
        except subprocess.TimeoutExpired:
            print("Unable to interpret array %s in file %s" %
                  (array_name, pkgbuild), file=sys.stderr)
            exit(1)

    if not cp.stdout:
        return []
    else:
        tmp = [line for line in cp.stdout.splitlines() if line]
        ret = []
        for t in tmp:
            ret += t.split()
        return list(set(ret))

def strip_version_info(package_name):
    """Return package_name without version numbers.

    Some packages specified as dependencies have a version number, e.g.
    gcc>=5.1. We shouldn't care about this, we always sync to an
    up-to-date mirror before building packages, so strip this info.
    """
    for pat in [r">=", r"<=", r"=", r"<", r">"]:
        depth = re.search(pat, package_name)
        if depth:
            package_name = package_name[:depth.start()]
    return package_name

def timestamp():
    return datetime.datetime.now().strftime("%Y-%m-%dT%H:%M:%S")

def log(kind, string, output=[], start_time=None):
    if kind not in ["command", "info", "die", "provide_info",
            "dep_info", "sloc_info", "red", "red_errors"]:
        raise RuntimeError("Bad kind: %s" % kind)

    if not start_time:
        start_time = timestamp()

    obj = {"time" : start_time,
         "kind" : kind,
         "head" : string,
         "body" : output
    }
    print(json.dumps(obj), file=sys.stderr)

class OutputDirectory():
    """Instantiated using the 'with' statement.

    This class is used to wrap the directory that a container script can
    write to inside a container. Calling it as follows:

    with OutputDirectory(args) as out_dir:
          ...

    means that container scripts do not have to worry about what
    directory they are supposed to be writing to. This class takes care
    of generating a unique output directory path, as well as updating a
    symlink to point to the latest result when the script container has
    finished executing.
    """
    def __init__(self, args):
        """Arguments:
            args: an argparse object returned from a call to
                  setup_argparser(). This object must have an attribute
                  called 'shared_directory'.
        """
        top_level = args.stage_name
        self.top_level = os.path.join(args.shared_directory,  top_level)

        timestamp = datetime.datetime.now().strftime("%Y-%m-%dT%H:%M:%S")
        self.path = os.path.join(self.top_level, timestamp)

    def __enter__(self):
        os.makedirs(self.path, exist_ok=True)
        return self.path

    def __exit__(self, type, value, traceback):
        latest = os.path.join(self.top_level, "latest")
        if os.path.lexists(latest):
            os.remove(latest)
        os.symlink(self.path, latest)

def get_argparser():
    """Set up command line options.

    Returns:
        an ArgumentParser, to which arguments can be appended before
        calling parse_args() to process them.
    """
    parser = argparse.ArgumentParser()
    parser.add_argument("--verbose",
                        dest="verbose", action="store_true")
    parser.add_argument("--output-directory",
                        dest="output_directory", action="store")

    parser.add_argument("--shared-directory",
                        dest="shared_directory", action="store")
    parser.add_argument("--shared-volume",
                        dest="shared_volume", action="store")

    parser.add_argument("--sources-directory",
                        dest="sources_directory", action="store")
    parser.add_argument("--sources-volume",
                        dest="sources_volume", action="store")

    parser.add_argument("--mirror-directory",
                        dest="mirror_directory", action="store")
    parser.add_argument("--mirror-volume",
                        dest="mirror_volume", action="store")

    parser.add_argument("--red-directory")

    parser.add_argument("--toolchain_root-directory")

    parser.add_argument("--stage-name",
                        dest="stage_name", action="store")

    parser.add_argument("--toolchain-directory",
                        dest="toolchain_directory", action="store")
    parser.add_argument("--toolchain-volume",
                        dest="toolchain_volume", action="store")

    parser.add_argument("--toolchain",
                        dest="toolchain", action="store")

    parser.add_argument("--env-vars", nargs="*",
                        dest="env_vars", action="store")

    return parser

def create_package(path, pkg_name, args):
    """Creates an Arch Linux package from the files in directory path.

    The files in path should include a .PKGINFO at the top-level; all
    standard Arch Linux packages should have one of these anyway. This
    method will create a .MTREE file in path (overwriting any existing
    .MTREE).

    Returns:
        the path to the new package, which will have extension
        .pkg.tar.xz.
    """
    if not os.path.lexists(os.path.join(path, ".PKGINFO")):
        raise RuntimeError("No .PKGINFO at " + path)

    owd = os.getcwd()
    os.chdir(path)

    log("info", "Generating .MTREE")
    try: os.unlink(".MTREE")
    except FileNotFoundError: pass
    files = " ".join(os.listdir("."))
    cmd = ("bsdtar -czf .MTREE --format=mtree"
           " --options=!all,use-set,type,uid,mode"
           ",time,size,md5,sha256,link " + files)
    time = timestamp()
    cp = subprocess.run(cmd.split(), stdout=subprocess.PIPE, stderr=subprocess.STDOUT,
             universal_newlines=True)
    log("command", cmd, cp.stdout.splitlines(), time)
    if cp.returncode:
        exit(1)

    log("info", "Tar-ing up files")
    pkg_name = pkg_name + ".pkg.tar.xz"
    files = " ".join(os.listdir("."))

    tar_cmd = "bsdtar -cf - " + files
    time = timestamp()
    tar_proc = subprocess.Popen(tar_cmd.split(), stdout=subprocess.PIPE, stderr=subprocess.PIPE,
                     universal_newlines=False)

    tar_data, tar_error = tar_proc.communicate()

    if tar_proc.returncode:
        log("command", tar_cmd, tar_error.decode("utf-8").splitlines(),
            time)
        exit(1)
    log("command", tar_cmd, [], time)

    xz_cmd = "xz -c -z"
    time = timestamp()
    xz_proc = subprocess.Popen(xz_cmd.split(), stdin=subprocess.PIPE, stdout=subprocess.PIPE,
                    stderr=subprocess.PIPE, universal_newlines=False)

    xz_data, xz_error = xz_proc.communicate(input=tar_data)

    if xz_proc.returncode:
        log("command", xz_cmd, xz_error.decode("utf-8").splitlines(),
            time)
        exit(1)
    log("command", xz_cmd, [], time)

    log("info", "Successfully ran " + tar_cmd + " | " + xz_cmd)
    with open(pkg_name, "bw") as f:
        f.write(xz_data)

    cmd = "bsdtar -tqf " + pkg_name + " .PKGINFO"
    time = timestamp()
    cp = subprocess.run(cmd.split(), stdout=subprocess.PIPE, stderr=subprocess.STDOUT,
             universal_newlines=True)
    log("command", cmd, cp.stdout.splitlines(), time)
    if cp.returncode:
        exit(1)

    pkg_path = os.path.join(args.toolchain_directory, pkg_name)
    try: os.remove(pkg_path)
    except: pass
    shutil.move(pkg_name, args.toolchain_directory)

    log("info", "Created package at path %s" % pkg_path)

    os.chdir(owd)
    return pkg_path

def toolchain_repo_name(): return "tuscan"

def recursive_chown(directory):
    """makepkg cannot be run as root.

    This function changes the owner and group owner of a directory tree
    rooted at directory to "tuscan".
    """
    shutil.chown(directory, "tuscan", "tuscan")
    for path, subdirs, files in os.walk(directory):
        for f in subdirs + files:
            shutil.chown(os.path.join(path, f), "tuscan", "tuscan")

def set_local_repository_location(path, repo_name):
    """Point pacman to a local repository.

    This function rewrites pacman.conf so that no remote mirrors are
    used to install packages; instead, the local repository specified
    will be used.

    Arguments:

        path: the absolute path to the directory where the repository
              database (and packages themselves) are located.

        repo_name: the name of the repository database file, without
                   extensions.

    This function will remove all repositories from pacman.conf and add
    the following directive:
    [repo_name]
    Server = file://path
    """

    # Point pacman to our toolchain repository. This means commenting
    # out the official repositories in pacman.conf, and creating a new
    # entry for the local toolchain repository.

    lines = []
    with open("/etc/pacman.conf") as f:
        appending = True
        for line in f:
            if re.search("# REPOSITORIES", line):
                lines.append(line.strip())
                appending = False
            if appending:
                lines.append(line.strip())

    lines.append("[%s]" % repo_name)
    lines.append("Server = file://%s" % path)

    with open("/etc/pacman.conf", "w") as f:
        f.write("\n".join(lines))

    log("info",
        "Removed vanilla repositories from pacman.conf and added:",
        lines[-2:])

    command = "pacman -Syy --noconfirm"
    time = timestamp()
    cp = subprocess.run(command.split(),
             stdout=subprocess.PIPE, stderr=subprocess.STDOUT,
             universal_newlines=True)
    log("command", command, cp.stdout.splitlines(), time)
    if cp.returncode:
        die(Status.failure)

def add_package_to_toolchain_repo(pkg, toolchain_repo_dir,
                                  remove_name=None):
    """Adds a package to the toolchain repository.

    This function is used when a hybrid package is created from a
    toolchain build. The hybrid package needs to be added to the local
    toolchain repository so that it can be installed if a future build
    depends on the package.

    Arguments:

        pkg: a path to a package (with extension .pkg.tar.xz)

        toolchain_repo_dir: directory where the toolchain repository is
                            stored, typically passed to stages via the
                            --toolchain-directory argument.

        remove_name: remove package with name 'remove_name' from
                     the repository immediately after adding package
                     pkg. This is intended to be used for creating a new
                     empty repository, by adding a 'fake' package and
                     then immediately removing it again.
    """
    if not os.path.isdir(toolchain_repo_dir):
        log("die", "Toolchain directory '%s' doesn't exist. "
                   "Check that the --tolchain-directory "
                   "argument was passed and the data-only "
                   "container was successfully created." %
                   (toolchain_repo_dir))

    repo = os.path.join(toolchain_repo_dir, toolchain_repo_name() + ".db.tar")

    # repo-add and -remove fail if they can't acquire a lock on the
    # database. This will happen quite often, as we're going to be
    # building many packages at once and adding them to the database.
    # So, keep trying to add until it works.

    random.seed()

    max_tries = 80
    attempt = 1
    while attempt <= max_tries:
        log("info",
            "Attempting to access local repository, attempt %d" %
                attempt)

        attempt = attempt + 1
        time.sleep(random.random() * 16)

        if attempt == max_tries:
            log("info", "Couldn't add %s to %s after %d tries, dying."
                % (pkg, repo, max_tries))
            exit(1)

        lock_fail = False

        cmd = "repo-add %s %s" % (repo, pkg)
        cp = subprocess.run(cmd.split(), stdout=subprocess.PIPE,
                stderr=subprocess.STDOUT, universal_newlines=True)
        log("command", cmd, cp.stdout.splitlines())

        for line in cp.stdout.splitlines():
            if re.search("ERROR: Failed to acquire lockfile", line):
                lock_fail = True
        if lock_fail:
            continue
        elif cp.returncode:
            exit(1)

        if not remove_name: return

        cmd = "repo-remove %s %s" % (repo, remove_name)
        cp = subprocess.run(cmd.split(), stdout=subprocess.PIPE,
                stderr=subprocess.STDOUT, universal_newlines=True)
        log("command", cmd, cp.stdout.splitlines())

        for line in cp.stdout.splitlines():
            if re.search("ERROR: Failed to acquire lockfile", line):
                lock_fail = True
        if lock_fail:
            continue
        elif cp.returncode:
            exit(1)

        return


from django.urls import path
from .views import Home,SettingsLov,SettingsAddLov,SettingsDeleteLov
from .views import TaskList,AddTask,DeleteTask,TaskDetail,TaskUpdate,AddTaskComment,DeleteTaskCommnet,AddLink,TaskCommentList,TaskCommentUpdate
from .views import ActivityDetail,DeleteActivity,ActivityList,AddActivity,ActivityUpdate,CommentsDetail #Activity Related Views
from . import views
from django.contrib.auth.views import LogoutView
from django.urls import path, include
from django.contrib.auth.decorators import login_required

urlpatterns = [
    path('tinymce/', include('tinymce.urls')),
    path('', views.login, name ="login"),
    path("logout/", LogoutView.as_view(template_name='projects/login.html'), name="logout"),
    path('Dashboard/', Home.as_view(),name="HomePage"),
    path('setting/lov/', SettingsLov.as_view(),name="lov-view"), #Lov List View
    path('setting/lov/add/', SettingsAddLov.as_view(), name='lov-add'), #Add Lov
    path('setting/lov/<pk>/delete/', SettingsDeleteLov.as_view(), name='lov-delete'), #Delete LOV
    path('task/', TaskList.as_view(),name="task-view"), #Task List View
    path('task/add/', AddTask.as_view(), name='task-add'), #Add Task
    path('task/<pk>/delete/', DeleteTask.as_view(), name='task-delete'), #Delete Task
    path('task/<slug:slug>/', TaskDetail.as_view(),name='task-detail'),
    path('task/shared/<str:signed_slug>/', views.get_shareable_link,name='shared-task-detail'),
    path('task/update/<pk>/', TaskUpdate.as_view(), name='task_update'),
    path('task/<pk>/comment/add/', AddTaskComment.as_view(), name='task-comment-add'), #Add Task comment
    path('task/comment/<pk>/delete/', DeleteTaskCommnet.as_view(), name='task-comment-delete'), #Delete Task comment
    path('link/add/', AddLink.as_view(), name='link-add'), #Add Link
    path('activity/add/', AddActivity.as_view(), name='activity-add'), #Add Activity
    path('activity/add/<pk>', AddActivity.as_view(), name='comment-activity-add'), #Add Activity from comment
    path('activity/', ActivityList.as_view(),name="activity-view"), #Activity List View
    path('activity/<slug:slug>', ActivityList.as_view(),name="activity-scheduled"), #Activity List View - scheduled
    path('activity/<pk>/', ActivityDetail.as_view(),name='activity-detail'),  #Detail of Activity
    path('activity/<pk>/delete/', DeleteActivity.as_view(), name='activity-delete'), #Delete Task
    path('activity/update/<pk>/', ActivityUpdate.as_view(), name='activity_update'),
    path('comments/', TaskCommentList.as_view(),name="task-comments-view"), #Task List View & used in tags serach
    path('comments/<slug:slug>/', TaskCommentList.as_view(), name="tagged"), #search on top banner
    path('comments/update/<pk>/', TaskCommentUpdate.as_view(), name='comment_update'),
    path('revision/', CommentsDetail.as_view(), name='revise-comments'), #it shows only one record
    path('revision/<pk>', CommentsDetail.as_view(), name='revise-update-comments'), #it shows only one record
    path('task/<slug:slug>/pdf/', login_required(views.generate_pdf), name='task_as_pdf'),
]


import socket
import sys
import time

from threading import Thread
from replicas_service import ReplicasService
from watcher_replicated_process import WatcherReplicatedProcess

PROCESS_ID = int(sys.argv[1])

# List of processes with their IDs and ports
processes = {1: 8001, 2: 8002, 3: 8003}

replicas_service = ReplicasService()
for process_id in processes.keys():
    replicated_process = WatcherReplicatedProcess(
        process_id, "localhost", processes[process_id]
    )
    if process_id == PROCESS_ID:
        replicas_service.set_my_process(replicated_process)
    else:
        replicas_service.add_process(replicated_process)

# Create TCP socket
print(f"[REPLICA {PROCESS_ID}] Listening on port {processes[PROCESS_ID]}")
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.bind(("0.0.0.0", processes[PROCESS_ID]))

# Start a thread to responde to messages
print(f"[REPLICA {PROCESS_ID}] Starting thread to respond messages...")
my_process = replicas_service.my_process
handler = Thread(
    target=my_process.response_to_messages, args=(s, replicas_service), daemon=True
)
handler.start()

# Start election to choose the leader
replicas_service.start_election()

# Main loop
while True:
    is_alive = True
    if replicas_service.get_leader() is None:
        print(f"[REPLICA {PROCESS_ID}] Leader not determined yet! Wating...")
    elif replicas_service.is_leader(replicas_service.get_my_process()):
        # Watch the proceses and restart them if they are not alive
        print("Doing some leader stuff")
    else:
        is_alive = replicas_service.get_leader().is_alive()

    if not is_alive:
        replicas_service.start_election()

    time.sleep(5)