from flask_wtf import FlaskForm
from wtforms import StringField, BooleanField, SelectField
from wtforms.validators import DataRequired
from wtforms.widgets import CheckboxInput
from flask_sqlalchemy import SQLAlchemy
from sqlalchemy.orm import aliased
from settings import admins, allreports, site_url, saml_sp, cookie_issuing_file
from datetime import datetime

db = SQLAlchemy()  # Create a database object

##################
# Object Classes #
##################

# Institution class
class Institution(db.Model):
    code = db.Column(db.String(255), primary_key=True)
    name = db.Column(db.String(255), nullable=False)
    fulfillment_code = db.Column(db.String(255), nullable=False, unique=True)
    partner_code = db.Column(db.String(255), nullable=False, unique=True)
    key = db.Column(db.String(255), nullable=True)
    exceptions = db.Column(db.String(255), nullable=True)
    ext_requests_in_transit = db.Column(db.String(255), nullable=True)
    in_transit_data = db.Column(db.String(255), nullable=True)

    # Constructor
    def __repr__(self):
        return '<Institution %r>' % self.code

    # Get institution's last update datetime
    def get_last_update(self):
        updated = db.session.execute(
            db.select(
                InstUpdate
            ).filter(
                InstUpdate.instcode == self.code
            ).order_by(
                InstUpdate.last_update.desc()
            )
        ).first()

        return updated

    # Get all current RequestExceptions statuses for the institution
    def get_statuses(self):
        statuses = db.session.execute(
            db.select(
                RequestException.borreqstat
            ).filter(
                RequestException.instcode == self.code
            ).group_by(
                RequestException.borreqstat
            ).order_by(
                RequestException.borreqstat
            )
        ).mappings().all()
        return statuses

    # Get all current RequestExceptions for the institution by status
    def get_exceptions_by_status(self, status):
        ib = aliased(Institution)
        il = aliased(Institution)
        requests = db.session.execute(
            db.select(
                RequestException.borreqstat, RequestException.internalid, RequestException.borcreate,
                RequestException.title, RequestException.author, RequestException.networknum,
                RequestException.partnerstat, RequestException.reqsend, RequestException.days,
                RequestException.requestor, RequestException.partnername, RequestException.partnercode,
                ExternalRequestInTransit.request_id, TransitStart.transit_date
            ).join(
                ib, RequestException.instcode == ib.code
            ).outerjoin(
                il, RequestException.partnercode == il.partner_code
            ).outerjoin(
                ExternalRequestInTransit, (ExternalRequestInTransit.title == RequestException.title) &
                                          (ExternalRequestInTransit.external_id == ib.fulfillment_code) &
                                          (ExternalRequestInTransit.instcode == il.code)
            ).outerjoin(
                TransitStart, ExternalRequestInTransit.request_id == TransitStart.request_id
            ).filter(
                RequestException.instcode == self.code,
                RequestException.borreqstat == status
            ).order_by(
                RequestException.borreqstat, RequestException.internalid.desc(), RequestException.borcreate.desc(),
                RequestException.reqsend.desc()
            )
        ).mappings().all()
        return requests

    # Get all current RequestExceptions for all institutions
    def get_all_requests(self):
        ib = aliased(Institution)
        il = aliased(Institution)
        requests = db.session.execute(
            db.select(
                RequestException.borreqstat.label('Borrowing Request Status'),
                RequestException.internalid.label('Internal ID'),
                RequestException.borcreate.label('Borrowing Request Date'),
                RequestException.title.label('Title'),
                RequestException.author.label('Author'),
                RequestException.networknum.label('Network Number'),
                RequestException.partnerstat.label('Partner Active Status'),
                RequestException.reqsend.label('Request Sending Date'),
                RequestException.days.label('Days Since Request'),
                RequestException.requestor.label('Requestor'),
                RequestException.partnername.label('Partner Name'),
                RequestException.partnercode.label('Partner Code'),
                ExternalRequestInTransit.request_id.label('In Transit?'),
                TransitStart.transit_date.label('In Transit Start')
            ).join(
                ib, RequestException.instcode == ib.code
            ).outerjoin(
                il, RequestException.partnercode == il.partner_code
            ).outerjoin(
                ExternalRequestInTransit, (ExternalRequestInTransit.title == RequestException.title) &
                                          (ExternalRequestInTransit.external_id == ib.fulfillment_code) &
                                          (ExternalRequestInTransit.instcode == il.code)
            ).outerjoin(
                TransitStart, ExternalRequestInTransit.request_id == TransitStart.request_id
            ).filter(
                RequestException.instcode == self.code,
            ).order_by(
                RequestException.borreqstat, RequestException.internalid.desc(), RequestException.borcreate.desc(),
                RequestException.reqsend.desc()
            )
        ).mappings().all()

        requests_dict = [dict(row) for row in requests]

        columns = ['Borrowing Request Status', 'Internal ID', 'Borrowing Request Date', 'Title', 'Author',
                   'Network Number',
                   'Requestor', 'Partner Active Status', 'Request Sending Date', 'Days Since Request', 'Partner Name',
                   'Partner Code', 'In Transit?', 'In Transit Start']

        parsed_requests = []  # List to hold parsed requests

        for request in requests_dict:  # Parse requests
            if request['In Transit?'] is None:  # If the request is not in transit
                request['In Transit?'] = 'N'  # Set the value to 'N'
            else:  # If the request is in transit
                request['In Transit?'] = 'Y'  # Set the value to 'Y'
            ordered_request = {k: request[k] for k in columns}  # Create a dictionary of the request
            parsed_requests.append(ordered_request)  # Add the parsed request to the list

        return parsed_requests  # Return the list of parsed requests

    # Get all current RequestExceptions for all institutions filtered by status
    def get_all_requests_filtered(self, userstatuses):
        ib = aliased(Institution)
        il = aliased(Institution)
        requests = db.session.execute(
            db.select(
                RequestException.borreqstat.label('Borrowing Request Status'),
                RequestException.internalid.label('Internal ID'),
                RequestException.borcreate.label('Borrowing Request Date'),
                RequestException.title.label('Title'),
                RequestException.author.label('Author'),
                RequestException.networknum.label('Network Number'),
                RequestException.partnerstat.label('Partner Active Status'),
                RequestException.reqsend.label('Request Sending Date'),
                RequestException.days.label('Days Since Request'),
                RequestException.requestor.label('Requestor'),
                RequestException.partnername.label('Partner Name'),
                RequestException.partnercode.label('Partner Code'),
                ExternalRequestInTransit.request_id.label('In Transit?'),
                TransitStart.transit_date.label('In Transit Start')
            ).join(
                ib, RequestException.instcode == ib.code
            ).outerjoin(
                il, RequestException.partnercode == il.partner_code
            ).outerjoin(
                ExternalRequestInTransit, (ExternalRequestInTransit.title == RequestException.title) &
                                          (ExternalRequestInTransit.external_id == ib.fulfillment_code) &
                                          (ExternalRequestInTransit.instcode == il.code)
            ).outerjoin(
                TransitStart, ExternalRequestInTransit.request_id == TransitStart.request_id
            ).filter(
                RequestException.instcode == self.code,
            ).filter(
                RequestException.borreqstat.any(userstatuses)
            ).order_by(
                RequestException.borreqstat, RequestException.internalid.desc(), RequestException.borcreate.desc(),
                RequestException.reqsend.desc()
            )
        ).mappings().all()

        requests_dict = [dict(row) for row in requests]

        columns = ['Borrowing Request Status', 'Internal ID', 'Borrowing Request Date', 'Title', 'Author',
                   'Network Number',
                   'Requestor', 'Partner Active Status', 'Request Sending Date', 'Days Since Request', 'Partner Name',
                   'Partner Code', 'In Transit?', 'In Transit Start']

        parsed_requests = []  # List to hold parsed requests

        for request in requests_dict:  # Parse requests
            if request['In Transit?'] is None:  # If the request is not in transit
                request['In Transit?'] = 'N'  # Set the value to 'N'
            else:  # If the request is in transit
                request['In Transit?'] = 'Y'  # Set the value to 'Y'
            ordered_request = {k: request[k] for k in columns}  # Create a dictionary of the request
            parsed_requests.append(ordered_request)  # Add the parsed request to the list

        return parsed_requests  # Return the list of parsed requests

# Institution form class
class InstitutionForm(FlaskForm):
    code = StringField('Code', validators=[DataRequired()])
    name = StringField('Name', validators=[DataRequired()])
    fulfillment_code = StringField('Fulfillment Code', validators=[DataRequired()])
    partner_code = StringField('Partner Code', validators=[DataRequired()])
    key = StringField('API Key')
    exceptions = StringField('Exceptions Path')
    ext_requests_in_transit = StringField('External Requests in Transit Path')
    in_transit_data = StringField('In Transit Data Path')

# Request Exception class
class RequestException(db.Model):
    exception_id = db.Column(db.BigInteger, primary_key=True)
    instcode = db.Column(db.ForeignKey(Institution.code))
    borreqstat = db.Column(db.String(255), nullable=True)
    internalid = db.Column(db.String(255), nullable=True)
    borcreate = db.Column(db.Date, nullable=True)
    title = db.Column(db.String(510), nullable=True)
    author = db.Column(db.String(255), nullable=True)
    networknum = db.Column(db.String(255), nullable=True)
    isbn = db.Column(db.String(255), nullable=True)
    issn = db.Column(db.String(255), nullable=True)
    requestor = db.Column(db.String(255), nullable=True)
    partnerstat = db.Column(db.String(255), nullable=True)
    reqsend = db.Column(db.DateTime, nullable=True)
    days = db.Column(db.Integer, nullable=True)
    partnername = db.Column(db.String(255), nullable=True)
    partnercode = db.Column(db.String(255), nullable=True)

    # Constructor
    def __repr__(self):
        return '<RequestException %r>' % self.exception_id

# External Request in Transit class
class ExternalRequestInTransit(db.Model):
    request_id = db.Column(db.BigInteger, primary_key=True)
    instcode = db.Column(db.ForeignKey(Institution.code))
    external_id = db.Column(db.String(255), nullable=False)
    requestor = db.Column(db.String(255), nullable=True)
    title = db.Column(db.String(510), nullable=True)
    author = db.Column(db.String(255), nullable=True)
    barcode = db.Column(db.String(255), nullable=True)
    physical_item_id = db.Column(db.String(255), nullable=True)
    isbn = db.Column(db.String(255), nullable=True)
    issn = db.Column(db.String(255), nullable=True)

    # Constructor
    def __repr__(self):
        return '<ExternalRequestInTransit %r>' % self.request_id

# Transit Start class
class TransitStart(db.Model):
    event_id = db.Column(db.BigInteger, primary_key=True)
    instcode = db.Column(db.ForeignKey(Institution.code))
    request_id = db.Column(db.String(255), nullable=False)
    transit_date = db.Column(db.DateTime, nullable=False)

    # Constructor
    def __repr__(self):
        return '<InTransitData %r>' % self.event_id

# Institution Update class
class InstUpdate(db.Model):
    id = db.Column(db.BigInteger, primary_key=True)
    instcode = db.Column(db.ForeignKey(Institution.code))
    last_update = db.Column(db.DateTime, nullable=False)

    # Constructor
    def __repr__(self):
        return '<InstUpdate %r>' % self.id

# User class
class User(db.Model):
    id = db.Column(db.Integer, primary_key=True)
    username = db.Column(db.String(255), nullable=False)
    displayname = db.Column(db.String(255), nullable=True)
    instcode = db.Column(db.ForeignKey(Institution.code))
    emailaddress = db.Column(db.String(255), nullable=True)
    admin = db.Column(db.Boolean, nullable=False)
    allreports = db.Column(db.Boolean, nullable=False)
    last_login = db.Column(db.DateTime, nullable=True)

    # Constructor
    def __repr__(self):
        return '<User %r>' % self.username

# User Day class
class UserDay(db.Model):
    id = db.Column(db.Integer, primary_key=True)
    user = db.Column(db.ForeignKey(User.id), nullable=False)
    day = db.Column(db.Integer, nullable=False)
    __table_args__ = (
        db.UniqueConstraint('user', 'day', name='_user_day_uc'),
    )

    # Constructor
    def __repr__(self):
        return '<UserDay %r>' % self.id

class StatusUser(db.Model):
    id = db.Column(db.Integer, primary_key=True)
    status = db.Column(db.String(255), nullable=False)
    user = db.Column(db.ForeignKey(User.id), nullable=False)
    __table_args__ = (
        db.UniqueConstraint('status', 'user', name='_status_user_uc'),
    )

    # Constructor
    def __repr__(self):
        return '<StatusUser %r>' % self.id

# User Settings form class
class UserSettingsForm(FlaskForm):
    sunday = BooleanField('Sunday', widget=CheckboxInput())
    monday = BooleanField('Monday', widget=CheckboxInput())
    tuesday = BooleanField('Tuesday', widget=CheckboxInput())
    wednesday = BooleanField('Wednesday', widget=CheckboxInput())
    thursday = BooleanField('Thursday', widget=CheckboxInput())
    friday = BooleanField('Friday', widget=CheckboxInput())
    saturday = BooleanField('Saturday', widget=CheckboxInput())
    AUTOMATIC_RENEW = BooleanField('Automatic renew', widget=CheckboxInput())
    AUTO_WILL_SUPPLY = BooleanField('Automatic will supply', widget=CheckboxInput())
    BAD_CITATION = BooleanField('Bad citation', widget=CheckboxInput())
    CANCEL_NOT_ACCEPTED = BooleanField('Cancel request not accepted', widget=CheckboxInput())
    CANCELLED = BooleanField('Cancelled by partner', widget=CheckboxInput())
    CLAIMED = BooleanField('Claimed', widget=CheckboxInput())
    CONDITIONAL = BooleanField('Conditional', widget=CheckboxInput())
    REQUEST_CREATED_BOR = BooleanField('Created borrowing request', widget=CheckboxInput())
    DAMAGED_COMMUNICATED = BooleanField('Damaged communicated', widget=CheckboxInput())
    DECLARED_LOST = BooleanField('Declared lost by partner', widget=CheckboxInput())
    RECEIVED_DIGITALLY = BooleanField('Digitally received by library', widget=CheckboxInput())
    EXPIRED = BooleanField('Expired', widget=CheckboxInput())
    EXTERNALLY_OBTAINED = BooleanField('Externally Obtained', widget=CheckboxInput())
    LENDER_CHECK_IN = BooleanField('Lender check in', widget=CheckboxInput())
    LOCAL_HOLDING = BooleanField('Local holding', widget=CheckboxInput())
    LOCATE_FAILED = BooleanField('Locate failed', widget=CheckboxInput())
    LOCATE_IN_PROCESS = BooleanField('Locate in process', widget=CheckboxInput())
    LOST_AND_FEE_PAID = BooleanField('Lost and fee paid', widget=CheckboxInput())
    LOST_COMMUNICATED = BooleanField('Lost communicated', widget=CheckboxInput())
    MANUAL_RENEW = BooleanField('Manual renew', widget=CheckboxInput())
    MEDIATED_RENEWAL = BooleanField('Mediated Patron Renewal', widget=CheckboxInput())
    OVERDUE_ITEM = BooleanField('Overdue request', widget=CheckboxInput())
    PENDING_APPROVAL = BooleanField('Pending Approval', widget=CheckboxInput())
    REACTIVATED = BooleanField('Reactivated', widget=CheckboxInput())
    READY_TO_SEND = BooleanField('Ready to be sent', widget=CheckboxInput())
    RECALLED_BOR = BooleanField('Recalled by partner', widget=CheckboxInput())
    REJECT = BooleanField('Reject', widget=CheckboxInput())
    RENEW_NOT_ACCEPTED = BooleanField('Renew request not accepted', widget=CheckboxInput())
    RENEW_REQUESTED = BooleanField('Renew requested', widget=CheckboxInput())
    RENEWED = BooleanField('Renewed by partner', widget=CheckboxInput())
    DAMAGED = BooleanField('Reported damaged item to partner', widget=CheckboxInput())
    REPORT_LOST = BooleanField('Reported lost item to partner', widget=CheckboxInput())
    REQUEST_ACCEPTED = BooleanField('Request accepted', widget=CheckboxInput())
    REQUEST_SENT = BooleanField('Request sent to partner', widget=CheckboxInput())
    RETURNED_TO_PARTNER = BooleanField('Returned item to partner', widget=CheckboxInput())
    SHIPPED_DIGITALLY = BooleanField('Shipped Digitally', widget=CheckboxInput())
    SHIPPED_PHYSICALLY = BooleanField('Shipped Physically', widget=CheckboxInput())
    CANCEL_REPLY = BooleanField('Waiting for cancel response', widget=CheckboxInput())
    RECEIVE_DIGITALLY_REPLY = BooleanField('Waiting for receive digitally', widget=CheckboxInput())
    WILL_SUPPLY = BooleanField('Will supply', widget=CheckboxInput())

# Login form class
class LoginForm(FlaskForm):
    institution = SelectField('Login At', choices=[
        ('au', 'American University'),
        ('amulaw', 'AU Washington College of Law'),
        ('cu', 'Catholic University of America'),
        ('ga', 'Gallaudet University'),
        ('gm', 'George Mason University'),
        ('gw', 'George Washington University'),
        ('gwalaw', 'GW Jacob Burns Law Library'),
        ('hi', 'GW Himmelfarb Health Sciences Library'),
        ('gt', 'Georgetown University'),
        ('gt-law', 'Georgetown University Law'),
        ('hu', 'Howard University'),
        ('mu', 'Marymount University'),
        ('dc', 'University of the District of Columbia'),
        ('wr', 'WRLC SCF'),
    ])

####################
# Helper Functions #
####################

# Add institution form submit
def add_institution_form_submit(form):
    institution = Institution(  # Build institution object from form data
        code=form.code.data, name=form.name.data, fulfillment_code=form.fulfillment_code.data,
        partner_code=form.partner_code.data, key=form.key.data, exceptions=form.exceptions.data,
        ext_requests_in_transit=form.ext_requests_in_transit.data, in_transit_data=form.in_transit_data.data
    )
    db.session.add(institution)  # Add institution to database
    db.session.commit()  # Commit changes to database

# Get all institutions
def get_all_institutions():
    institutions = db.session.execute(db.select(Institution).order_by(Institution.name)).scalars()
    return institutions

# Construct login URL
def construct_login_url(institution):
    login_url = saml_sp
    login_url += cookie_issuing_file
    login_url += '?institution='
    login_url += institution
    login_url += '&url='
    login_url += site_url
    login_url += '/login/n'
    return login_url

# Log the user in
def user_login(session, user_data):

    # Set the session variables
    session['username'] = user_data['primary_id']  # Set the username
    session['user_home'] = user_data['inst']  # Set the user's home institution
    session['display_name'] = user_data['full_name']  # Set the user's display name
    session['authorizations'] = user_data['authorizations']  # Set the user's authorizations
    session['email'] = user_data['email']  # Set the user's email

    user = check_user(session['username'])  # Check if the user exists in the database

    # If the user is in the database...
    if user is not None:
        set_email(user, session)  # ...set the user's email address
        set_user_admin(user, session)  # ...set the user's admin status
        set_user_allreports(user, session)  # ...set the user's allreports status
        if 'exceptions' in session['authorizations']:
            set_last_login(user)  # ...set the last login time for the user

    # If the user isn't in the database...
    else:
        admincheck = admincheck_user(session)  # ...check if the user is an admin
        allreportscheck = allreportscheck_user(session)  # ...check if the user is an admin
        add_user(session, admincheck, allreportscheck)  # ...add the user to the database
        set_user_admin(user, session)  # ...set the user's admin status
        set_user_allreports(user, session)  # ...set the user's allreports status

# Check if the user exists in the database
def check_user(username):
    user = db.session.execute(db.select(User).filter(User.username == username)).scalar_one_or_none()
    return user

# Set the user's email address
def set_email(user, session):
    user.emailaddress = session['email']
    db.session.commit()

# Set the last login time for the user
def set_last_login(user):
    user.last_login = datetime.now()  # Set the last login time to the current time
    db.session.commit()  # Commit the changes

# Set the user's admin status based on the database
def set_user_admin(user, session):
    if user.admin is True:  # Check if the user is an admin
        if 'exceptions' not in session['authorizations']:
            session['authorizations'].append('exceptions')  # If they are, add the admin authorization to the session
        session['authorizations'].append('admin')  # If they are, add the admin authorization to the session

# Check if the username is in the admin list
def admincheck_user(session):
    if session['username'] in admins:  # Check if the user is an admin
        admincheck = True  # If they are, set admincheck to True
    else:
        admincheck = False  # If they are not, set admincheck to False

    return admincheck

# Set the user's allreport status based on the database
def set_user_allreports(user, session):
    if user.allreports is True:  # Check if the user is an admin
        if 'exceptions' not in session['authorizations']:
            session['authorizations'].append('exceptions')  # If they are, add the admin authorization to the session
        session['authorizations'].append('allreports')  # If they are, add the admin authorization to the session

# Check if the username is in the allreports list
def allreportscheck_user(session):
    if session['username'] in allreports:  # Check if the user is an admin
        allreportscheck = True  # If they are, set admincheck to True
    else:
        allreportscheck = False  # If they are not, set admincheck to False

    return allreportscheck

# Add the user to the database
def add_user(session, admincheck, allreportscheck):

    # Create the user object
    user = User(
        username=session['username'],
        displayname=session['display_name'],
        instcode=session['user_home'],
        emailaddress=session['email'],
        admin=admincheck,
        allreports=allreportscheck,
        last_login=datetime.now()
    )
    db.session.add(user)  # Add the user to the database
    db.session.commit()  # Commit the changes

# Get all last updated times for institutions
def get_all_last_updates():
    updates = db.session.execute(
        db.select(
            InstUpdate.instcode, db.func.max(InstUpdate.last_update).label('last_update')
        ).group_by(InstUpdate.instcode)
    ).all()

    return updates

# Update user settings
def update_user_settings(form, user):
    db.session.execute(db.delete(UserDay).where(UserDay.user == user.id))  # delete all the user's days
    days = [
        (0, form.data['sunday']),
        (1, form.data['monday']),
        (2, form.data['tuesday']),
        (3, form.data['wednesday']),
        (4, form.data['thursday']),
        (5, form.data['friday']),
        (6, form.data['saturday']),
    ]
    for day in days:  # for each day
        if day[1] is True:
            user_day = UserDay(user=user.id, day=day[0])  # create a new user day
            db.session.add(user_day)  # add the user day to the database

    db.session.execute(db.delete(StatusUser).where(StatusUser.user == user.id))  # delete all the user's statuses
    statuses = [
        ('AUTOMATIC_RENEW', form.data['AUTOMATIC_RENEW']),
        ('AUTO_WILL_SUPPLY', form.data['AUTO_WILL_SUPPLY']),
        ('BAD_CITATION', form.data['BAD_CITATION']),
        ('CANCEL_NOT_ACCEPTED', form.data['CANCEL_NOT_ACCEPTED']),
        ('CANCELLED', form.data['CANCELLED']),
        ('CLAIMED', form.data['CLAIMED']),
        ('CONDITIONAL', form.data['CONDITIONAL']),
        ('REQUEST_CREATED_BOR', form.data['REQUEST_CREATED_BOR']),
        ('DAMAGED_COMMUNICATED', form.data['DAMAGED_COMMUNICATED']),
        ('DECLARED_LOST', form.data['DECLARED_LOST']),
        ('RECEIVED_DIGITALLY', form.data['RECEIVED_DIGITALLY']),
        ('EXPIRED', form.data['EXPIRED']),
        ('EXTERNALLY_OBTAINED', form.data['EXTERNALLY_OBTAINED']),
        ('LENDER_CHECK_IN', form.data['LENDER_CHECK_IN']),
        ('LOCAL_HOLDING', form.data['LOCAL_HOLDING']),
        ('LOCATE_FAILED', form.data['LOCATE_FAILED']),
        ('LOCATE_IN_PROCESS', form.data['LOCATE_IN_PROCESS']),
        ('LOST_AND_FEE_PAID', form.data['LOST_AND_FEE_PAID']),
        ('LOST_COMMUNICATED', form.data['LOST_COMMUNICATED']),
        ('MANUAL_RENEW', form.data['MANUAL_RENEW']),
        ('MEDIATED_RENEWAL', form.data['MEDIATED_RENEWAL']),
        ('OVERDUE_ITEM', form.data['OVERDUE_ITEM']),
        ('PENDING_APPROVAL', form.data['PENDING_APPROVAL']),
        ('REACTIVATED', form.data['REACTIVATED']),
        ('READY_TO_SEND', form.data['READY_TO_SEND']),
        ('RECALLED_BOR', form.data['RECALLED_BOR']),
        ('REJECT', form.data['REJECT']),
        ('RENEW_NOT_ACCEPTED', form.data['RENEW_NOT_ACCEPTED']),
        ('RENEW_REQUESTED', form.data['RENEW_REQUESTED']),
        ('RENEWED', form.data['RENEWED']),
        ('DAMAGED', form.data['DAMAGED']),
        ('REPORT_LOST', form.data['REPORT_LOST']),
        ('REQUEST_ACCEPTED', form.data['REQUEST_ACCEPTED']),
        ('REQUEST_SENT', form.data['REQUEST_SENT']),
        ('RETURNED_TO_PARTNER', form.data['RETURNED_TO_PARTNER']),
        ('SHIPPED_DIGITALLY', form.data['SHIPPED_DIGITALLY']),
        ('SHIPPED_PHYSICALLY', form.data['SHIPPED_PHYSICALLY']),
        ('CANCEL_REPLY', form.data['CANCEL_REPLY']),
        ('RECEIVE_DIGITALLY_REPLY', form.data['RECEIVE_DIGITALLY_REPLY']),
        ('WILL_SUPPLY', form.data['WILL_SUPPLY']),
    ]
    for status in statuses:  # for each status
        if status[1] is True:
            status_user = StatusUser(status=status[0], user=user.id)  # create a new status user
            db.session.add(status_user)  # add the status user to the database

    db.session.commit()  # commit changes to the database


import numpy as np

from typing import List, Union, Dict
from pathlib import Path

import os
import torch
import logging
import flair
import gensim

from flair.data import Sentence
from flair.embeddings.token import TokenEmbeddings
from flair.embeddings import StackedEmbeddings
from flair.embeddings import WordEmbeddings, CharacterEmbeddings
from flair.file_utils import cached_path

log = logging.getLogger("flair")

def get_embedding(embedding, finetune_bert=False):
    embeddings = embedding.split('+')
    result = [CaseEmbedding()]
    # skip updating to new flair version
    old_base_path = "https://flair.informatik.hu-berlin.de/resources/embeddings/token/"
    cache_dir = Path("embeddings")
    cached_path(f"{old_base_path}glove.gensim.vectors.npy", cache_dir=cache_dir)
    cached_path(
        f"{old_base_path}glove.gensim", cache_dir=cache_dir
    )

    cached_path(f"https://flair.informatik.hu-berlin.de/resources/characters/common_characters", cache_dir="datasets")

    for embedding in embeddings:
        if embedding == 'char':
            result.append(CustomCharacterEmbeddings())
        if embedding == 'glove':
            result.append(LargeGloveEmbeddings('./data/glove'))

    return StackedEmbeddings(embeddings=result)

class LargeGloveEmbeddings(WordEmbeddings):
    """Standard static word embeddings, such as GloVe or FastText."""

    def __init__(self, glove_dir):
        """
        Initializes classic word embeddings - made for large glove embedding
        """

        super().__init__('glove')
        embeddings = '840b-300d-glove'
        self.field = ""
        self.embeddings = embeddings
        self.static_embeddings = True

        # Large Glove embeddings
        embeddings = os.path.join(glove_dir, 'glove.bin')
        self.name: str = str(embeddings)
        self.precomputed_word_embeddings = gensim.models.KeyedVectors.load(embeddings)
        self.__embedding_length: int = self.precomputed_word_embeddings.vector_size

    def _add_embeddings_internal(self, sentences: List[Sentence]) -> List[Sentence]:

        for i, sentence in enumerate(sentences):
            for token, token_idx in zip(sentence.tokens, range(len(sentence.tokens))):
                word_embedding = self.get_cached_vec(word=token.text)
                token.set_embedding(self.name, word_embedding)

        return sentences

    @property
    def embedding_length(self) -> int:
        return 300

class CaseEmbedding(TokenEmbeddings):
    """Static Case Embedding"""

    def __init__(self):
        self.name: str = 'case-embedding-shun'
        self.static_embeddings = False
        self.__embedding_length: int = 3
        super().__init__()

    @property
    def embedding_length(self) -> int:
        return self.__embedding_length

    def embed(self, sentences: Union[Sentence, List[Sentence]]) -> List[Sentence]:
        for sentence in sentences:
            for token in sentence:
                text = token.text
                is_lower = 1 if text == text.lower() else 0
                is_upper = 1 if text == text.upper() else 0
                is_mix = 1 if is_lower + is_upper == 0 else 0
                word_embedding = torch.tensor(
                    np.array([is_lower, is_upper, is_mix]), device=flair.device, dtype=torch.float
                )
                token.set_embedding('case-embedding-shun', word_embedding)

        return sentences

    def _add_embeddings_internal(self, sentences: List[Sentence]) -> List[Sentence]:
        return sentences

    def __str__(self):
        return self.name

class CustomCharacterEmbeddings(CharacterEmbeddings):
    """Batched-version of CharacterEmbeddings. """

    def _add_embeddings_internal(self, sentences: List[Sentence]):

        token_to_embeddings = {}

        for sentence in sentences:
            for token in sentence.tokens:
                token_to_embeddings[token.text] = None

        tokens_char_indices = []
        for token in token_to_embeddings:
            char_indices = [
                self.char_dictionary.get_idx_for_item(char) for char in token
            ]
            tokens_char_indices.append(char_indices)

        # sort words by length, for batching and masking
        tokens_sorted_by_length = sorted(
            tokens_char_indices, key=lambda p: len(p), reverse=True
        )
        d = {}
        for i, ci in enumerate(tokens_char_indices):
            for j, cj in enumerate(tokens_sorted_by_length):
                if ci == cj:
                    d[j] = i
                    continue
        chars2_length = [len(c) for c in tokens_sorted_by_length]
        longest_token_in_sentence = max(chars2_length)
        tokens_mask = torch.zeros(
            (len(tokens_sorted_by_length), longest_token_in_sentence),
            dtype=torch.long,
            device=flair.device,
        )

        for i, c in enumerate(tokens_sorted_by_length):
            tokens_mask[i, : chars2_length[i]] = torch.tensor(
                c, dtype=torch.long, device=flair.device
            )

        # chars for rnn processing
        chars = tokens_mask

        character_embeddings = self.char_embedding(chars).transpose(0, 1)

        packed = torch.nn.utils.rnn.pack_padded_sequence(
            character_embeddings, chars2_length
        )

        lstm_out, self.hidden = self.char_rnn(packed)

        outputs, output_lengths = torch.nn.utils.rnn.pad_packed_sequence(lstm_out)
        outputs = outputs.transpose(0, 1)
        chars_embeds_temp = torch.zeros(
            (outputs.size(0), outputs.size(2)),
            dtype=torch.float,
            device=flair.device,
        )
        for i, index in enumerate(output_lengths):
            chars_embeds_temp[i] = outputs[i, index - 1]
        character_embeddings = chars_embeds_temp.clone()
        for i in range(character_embeddings.size(0)):
            character_embeddings[d[i]] = chars_embeds_temp[i]

        for token_number, token in enumerate(token_to_embeddings.keys()):
            token_to_embeddings[token] = character_embeddings[token_number]

        for sentence in sentences:
            for token in sentence.tokens:
                token.set_embedding(self.name, token_to_embeddings[token.text])


from __future__ import print_function
import sys, readline
from os import environ

class InvalidCommandException(Exception):
    pass

class Autocomplete(object):
    def __init__(self, options):
        self.options = sorted(options)
    def complete(self, text, state):
        if state == 0:
            if not text:
                self.matches = self.options[:]
            else:
                self.matches = [s for s in self.options if s and s.startswith(text)]
        try:
            return self.matches[state]
        except IndexError:
            return None
    def display_matches(self, subsitution, matches, longest_match_length):
        line_buffer = readline.get_line_buffer()
        columns = environ.get("COLUMNS", 80)
        print()
        tpl = "{:<" + str(int(max(map(len, matches)) * 1.2)) + "}"
        buffer = ""
        for match in matches:
            match = tpl.format(match[len(subsitution):])
            if len(buffer + match) > columns:
                print(buffer)
                buffer = ""
            buffer += match
        if buffer:
            print(buffer)
        print("> ",end="")
        sys.stdout.flush()

class ExampleHandler(object):
    def __init__(self):
        self.commands = {
            'help': 'view help menu',
            'exit': 'quits the shell'
        }
    def handle(self, cmd):
        if cmd == 'help':
            print('\n'.join(['{}: {}'.format(k, v) for k, v in self.commands.items()]))
        elif cmd == "exit":
            return "exit"
        else:
            raise InvalidCommandException("Command not found: {}".format(cmd))

def start_shell(cmd_handler):
    commands = cmd_handler.commands.keys()
    completer = Autocomplete(list(set(commands)))
    readline.set_completer_delims('\t')
    readline.set_completer(completer.complete)
    readline.parse_and_bind('tab: complete')
    readline.set_completion_display_matches_hook(completer.display_matches)
    while True:
        cmd = input("> ")
        try:
            res = cmd_handler.handle(cmd)
        except Exception as e:
            print("Error: "+str(e))
            res = None
        if res == "exit":
            break


# https://music.douban.com/tag/
# 5-6个标签
# 歌曲名 歌手 日期 评分 评价人数
import time

import requests, csv
from bs4 import BeautifulSoup
from pyecharts.charts import Pie, Bar, Scatter3D, Scatter, Line
from pyecharts import options as opts

tags = ['OST', '民谣', '流行', 'pop', 'indie', 'Electronic', 'Folk', '摇滚', 'J-POP', 'rock', '电影原声', 'R&B','中国摇滚','纯音乐'][:9]
header = {
    'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.132 Safari/537.36 QIHU 360SE'
}
url_demo = 'https://music.douban.com/tag/{}?start={}&type=T'

# 获取html源代码
def get_html(url):
    r = requests.get(url, headers=header)
    html = r.text
    return html

# 找class=pl2下的a的内容
def get_song_name(soup):
    div = soup.find('div', attrs={'class': "pl2"})
    a = div.find('a')
    if a:
        return a.text.replace(' ', '').replace('\n', '')
    return '空标题'

# 找class=pl2下的p的内容/分割第一个
def get_song_author(soup):
    div = soup.find('div', attrs={'class': "pl2"})
    p = div.find('p')
    if p:
        texts = p.text.split('/')
        if len(texts) != 0:
            return texts[0].strip()
    return '空作者'

# 找class=pl2下p的/分割第2个
def get_song_date(soup):
    div = soup.find('div', attrs={'class': "pl2"})
    p = div.find('p')
    if p:
        texts = p.text.split('/')
        if len(texts) >= 2:
            return texts[1].strip()
    return '空时间'

# 找class=rating_nums 的span的内容
def get_song_rank(soup):
    span = soup.find('span', attrs={'class': "rating_nums"})
    if span:
        return span.text
    return '空评分'

# 找class=pl2下的class=pl的span
def get_song_comment_count(soup):
    div = soup.find('div', attrs={'class': "pl2"})
    if div:
        span = div.find('span', attrs={'class': 'pl'})
        if span:
            # 替换无效的字符
            return span.text.replace(' ', '').replace('(', '').replace(')', '').replace('\n', '')
    return '空评论人数'

# 根据偏移量获取数据
def get_songs_by_offset(tag, page):
    songs = []
    # 设置完整url
    url = url_demo.format(tag, page)
    print(url)
    # 获取html源码
    html = get_html(url)
    soup = BeautifulSoup(html, 'lxml')
    # 找class=item 的tr数组
    items = soup.find_all('tr', attrs={'class': 'item'})
    # 没找到返回空
    if len(items) == 0:
        return songs
    # 遍历每一个tr
    for item in items:
        # 传tr，让函数获取其中的内容
        name = get_song_name(item)
        author = get_song_author(item)
        date = get_song_date(item)
        rank = get_song_rank(item)
        comment_count = get_song_comment_count(item)
        # 组合成一个数据
        song = (name, author, date, rank, comment_count)
        print(song)
        # 添加每一条，到一页数据的数组
        songs.append(song)
    return songs

# 根据tag采集内容
def get_songs_by_tag(tag):
    offset = 0 #初始偏移量0
    page = 1 #初始页数
    while True:
        print("正在采集", page, '页')
        #获取每页数据
        songs_ = get_songs_by_offset(tag, offset)
        if len(songs_) == 0:
            print('tag采集结束')
            break
        print("第{}页采集成功".format(page))
        # 将数据写入文件
        write_many_to_csv(songs_, tag)
        # 下一页的偏差
        offset += 20
        page += 1

        if page == 2: #采集两页测试
            break
        #程序暂停5秒
        time.sleep(5)

def write_many_to_csv(songs, tag):
    #打开csv文件
    with open(tag + '_songs.csv', 'a', encoding='utf8') as f:
        writer = csv.writer(f)
        #将数组写入csv
        writer.writerows(songs)

#获取全部数据
def get_songs():
    for tag in tags:
        print("采集", tag)
        get_songs_by_tag(tag)

#根据tag获取文件内容
def read(tag):
    data = []
    with open(tag + '_songs.csv', 'r', encoding='utf-8-sig') as f:
        reader = csv.reader(f)
        for item in reader:
            if item:
                data.append(item)
    return data

def draw1():
    counts = {}
    # 遍历每一个tag
    for tag in tags:
        #获取每一个tag的数据
        data = read(tag)
        #获取数据的长度
        count = len(data)
        #记录每个tag有多少数据
        counts[tag] = count
    #生成图
    Pie().add('', list(counts.items())).render('1.html')

def filt(x):
    try:
        #将x转为数字，失败则返回0
        return float(x)
    except:
        return 0

def draw2():
    data_ = []
    for tag in tags:
        #根据tag获取数据
        data = read(tag)
        #提取数据中的名字和评论数，组成新数据
        data_.extend([(x[0], filt(x[-1][:-3])) for x in data])
    #按评论数排序
    data = sorted(data_, key=lambda x: x[-1], reverse=True)[:30]
    #生成图
    bar = Bar()
    bar.add_xaxis([item[0] for item in data])
    bar.add_yaxis("评论数", [item[1] for item in data])
    bar.set_global_opts(title_opts=opts.TitleOpts(title="歌曲评论"))
    bar.render('2.html')

def draw3():
    data_ = []
    for tag in tags:
        # 根据tag获取数据
        data = read(tag)
        # 提取数据中的名字和评论数，组成新数据
        data_.extend([(x[0], filt(x[-1][:-3])) for x in data])
    # 按评论数排序
    data = sorted(data_, key=lambda x: x[-1], reverse=True)[:30]
    # 生成图
    Scatter().add_xaxis(xaxis_data=[item[0] for item in data]).add_yaxis(
        series_name="",
        y_axis=[item[1] for item in data],
        symbol_size=20,
        label_opts=opts.LabelOpts(is_show=False),
    ).render('3.html')

def draw4():
    data_ = []
    for tag in tags:
        # 根据tag获取数据
        data = read(tag)
        # 提取数据中的名字和评论数，组成新数据
        data_.extend([(x[0], filt(x[-1][:-3])) for x in data])
    # 按评论数排序
    data = sorted(data_, key=lambda x: x[-1], reverse=True)[:30]
    # 生成图
    Line().add_xaxis(xaxis_data=[item[0] for item in data]).add_yaxis(
        series_name="",
        y_axis=[item[1] for item in data],
        symbol="emptyCircle",
        is_symbol_show=True,
        label_opts=opts.LabelOpts(is_show=False),
        areastyle_opts=opts.AreaStyleOpts(opacity=1, color="#C67570"),
    ).render('4.html')

def draw5():
    data_ = []
    for tag in tags:
        # 根据tag获取数据
        data =read(tag)
        # 提取数据中的名字和时间，组成新数据
        data_.extend([(x[0] ,x[2]) for x in data])
    #根据时间排序
    data = sorted(data_, key=lambda x: x[-1], reverse=True)
    #日期计数
    dic = ['2000-01-01', '2001-01-01', '2002-01-01', '2003-01-01', '2004-01-01',
           '2005-01-01', '2006-01-01','2007-01-01', '2008-01-01', '2009-01-01',
           '2010-01-01', '2011-01-01','2012-01-01', '2013-01-01', '2014-01-01',
           '2015-01-01', '2016-01-01','2017-01-01', '2018-01-01', '2019-01-01','2020-01-01','2021-01-01']
    count_ = {}
    for item in data:
        for di in dic:
            #获取每一个日期
            if item[-1] < di:
                #日期计数
                if di not in count_:
                    count_[di] = 0
                else:
                    count_[di] += 1
                break
    #生成图
    Pie().add('', list(count_.items())).render('5.html')

if __name__ == '__main__':
    get_songs()
    draw1()
    draw2()
    draw3()
    draw4()
    draw5()


from rest_framework import serializers
from .models import Moto

class MotoSerializer(serializers.ModelSerializer):
    class Meta:
        model = Moto
        fields = '__all__'

    def to_internal_value(self, data):
        for key, value in data.items():
            if value == "":
                data[key] = None
        return super(MotoSerializer, self).to_internal_value(data)

class Geeks(object):
    def __init__(self, dictionary):
        self.dict = dictionary

# create a serializer
class GeeksSerializer(serializers.Serializer):
    # initialize fields
    dictionary = serializers.DictField(
    child = serializers.CharField())


# -*- coding: utf-8 -*-
import json
import logging

from core.spiders.spider_man import SpiderMan

from .item import GlobalItem

logger = logging.getLogger(__name__)

class GlobalSpider(SpiderMan):
    name = 'global_statistics'
    allowed_domains = ['coinmarketcap.com']
    url = 'https://s2.coinmarketcap.com/generated/stats/global.json'
    market = 'coinmarketcap'
    custom_settings = {
        'ITEM_PIPELINES': {
            'core.pipelines.RepositoryPipeline': 2
        }
    }

    def parse(self, response):
        logger.info('Global spider is running...')
        json_response = json.loads(response.body_as_unicode())
        item = GlobalItem()
        item["active_cryptocurrencies"] = json_response["active_cryptocurrencies"]
        item["active_markets"] = json_response["active_markets"]
        item["bitcoin_percentage_of_market_cap"] = json_response["bitcoin_percentage_of_market_cap"]
        item["total_market_cap_by_available_supply_usd"] = json_response["total_market_cap_by_available_supply_usd"]
        item["total_volume_usd"] = json_response["total_volume_usd"]
        yield item


import os
import sys
import pickle
import pandas as pd
import dask.dataframe as dd
import numpy as np
import xgboost as xgb

from sklearn.multiclass import OneVsRestClassifier
from sklearn.svm import LinearSVC
from sklearn.metrics import confusion_matrix, classification_report
from sklearn.preprocessing import LabelEncoder
from sklearn.model_selection import train_test_split
from sklearn.feature_extraction.text import TfidfVectorizer

DATA_PATH = '../../data/balanced-data.csv'
TFIDF_PATH = './tfidf.pkl'
THEMES = [5, 6, 33, 163, 181, 188, 232, 313, 339, 350, 406, 409, 424, 555, 589,
          597, 634, 660, 695, 766, 773, 793, 800, 810, 852, 895, 951]

def groupby_process(df):
    new_df = df.sort_values(['process_id', 'page'])
    new_df = new_df.groupby(
                ['process_id', 'themes'],
                group_keys=False
            ).apply(lambda x: x.body.str.cat(sep=' ')).reset_index()
    new_df = new_df.rename(index=str, columns={0: "body"})
    return new_df

def get_data(path):
    data = pd.read_csv(path)
    data = data.rename(columns={'document_id': 'process_id', 'pages': 'page'})
    #print(data.head())
    return data

def predict(dtest, clf):
    pred = clf.predict(dtest)
    return pred

def split_data(data):
    data.themes = data.themes.apply(lambda x: 'Theme ' + str(x))
    train_body, test_body, train_labels, test_labels = train_test_split(
        data.body,
        data.themes,
        test_size=0.33,
        random_state=42,
        stratify=data.themes
    )

    return train_body, test_body, train_labels, test_labels

def tfidf(train_body, test_body):
    tfidf = TfidfVectorizer(
        min_df=0.2,
        max_df=0.9,
        lowercase=False,
        use_idf=1,
        smooth_idf=1,
        #  sublinear_tf=1,
        # max_features=20000
    )

    X_train = tfidf.fit_transform(train_body)
    X_test = tfidf.transform(test_body)

    print('\tX_train', X_train.shape)
    print('\t\tX_test', X_test.shape)

    return X_train, X_test

def tfidf_from_file(path, body):
    with open(path, 'rb') as f:
        tfidf = pickle.load(f)

    transformed = tfidf.transform(body)
    return transformed

def transform_y(train_labels, test_labels):
    le = LabelEncoder()
    le.fit(train_labels)

    le_train = le.transform(train_labels)
    le_test = le.transform(test_labels)

    print(le.classes_)

    return le_train, le_test, le

def report(y_true, pred):
    try:
        print(classification_report(y_true, pred))
        # print(confusion_matrix(y_true, pred))
    except Exception as e:
        print(e)

if __name__ == '__main__':
    #  dtrain, dtest = get_data('./dtrain', './dtest')

    CHUNKS = 30000

    data = get_data(DATA_PATH)
    #print(data.head())

    print('Values counts: \n', data.themes.value_counts())

    train_body, test_body, train_labels, test_labels = split_data(data)
    #print(train_labels.head(100))

    print('Chunk size: {}'.format(CHUNKS))

    try:
        X_train, X_test = tfidf(train_body, test_body)
    except Exception as e:
        print(e)

    y_train, y_test, le = transform_y(train_labels, test_labels)

    print('We\'re classifying {} themes!'.format(len(le.classes_)))

    print('Started Training!')
    svc = OneVsRestClassifier(
            LinearSVC(),
            n_jobs=-1
            )

    svc.fit(X_train, y_train)
    print("Finished training")

    print('deleting obsolette vars')
    del X_train, y_train

    print('Starting predictions on dtest!')
    pred = svc.predict(X_test)

    print(y_test[:5], pred[:5])
    print(pred.max(), pred.min(), pred.mean(), pred.sum())
    #print(le.inverse_transform(y_test))
    #print(le.inverse_transform(pred.astype(int)))
    report(le.inverse_transform(y_test), le.inverse_transform(pred.astype(int)))


# -*- coding: utf-8 -*-

import pytest
from mock import Mock
from mock import patch

import os

from csvToJsonConverter import csvToJsonConverter

class TestCsvToJsonConverter():

    # def teardown_method(self, method):
    #     #  teardown_method is invoked after every test method of a class
    #     self.csv_file.close()

    def setup_method(self, method):
        #  setup_method is invoked befor every test method of a class
        self.converter = csvToJsonConverter()

    def test_complete_run(self):
        result_path = "testData/test_result.json"
        try:
            os.remove(result_path)
        except OSError:
            pass
        assert os.path.isfile(result_path) == False
        self.converter.run("testData/test_for_jsonConverter.csv", result_path)
        assert os.path.isfile(result_path) == True
        with open("testData/expected_result.json", 'r') as expected_result_file:
            expected_result = expected_result_file.read()
        with open(result_path, 'r') as result_file:
            result = result_file.read()
        assert expected_result == result

    def test_run(self):
        self.converter.create_dump_list = Mock(return_value=[{"dumpListElement1"}, {"dumpListElement2"}])
        self.converter.write_dump_list_to_file = Mock()
        self.converter.run("csvPath", "jsonPath")
        self.converter.create_dump_list.assert_called_once_with("csvPath")
        self.converter.write_dump_list_to_file.assert_called_once_with([{"dumpListElement1"}, {"dumpListElement2"}], "jsonPath")


import os

import requests
from django.contrib.auth.models import User
from django.core import validators
from django.core.files.base import ContentFile
from django.db import models

class Post(models.Model):
    url = models.URLField(max_length=200, default="", blank=True)
    title = models.CharField(max_length=200,blank=True)
    cover = models.FileField(upload_to="pdf/", validators=[validators.FileExtensionValidator(['pdf'],
	     message='file must be pdf')],blank=False)
    # cover = models.ImageField(upload_to="images/", default=None, blank=True)
    created = models.DateTimeField(auto_now_add=True)
    user = models.ForeignKey(User, on_delete=models.CASCADE, default=None)

    def save(self, *args, **kwargs):
        if self.url and not self.cover:
            response = requests.get(self.url)
            if response.status_code == 200:
                self.cover.save(os.path.basename(self.url), ContentFile(response.content), save=True)
            else:
                pass
        if self.title=='':
            self.title = self.cover.name
        super().save(*args, **kwargs)

    def __str__(self):
        return self.title

class Operation(models.Model):
    created = models.DateTimeField(auto_now_add=True)
    type = models.CharField(max_length=200)
    post = models.ForeignKey(Post, on_delete=models.CASCADE, default=None)


import os
import numpy as np
import torch
import math
from scipy import io
from construct import get_Pearson_fc,get_BOLD_feature,get_Degree_fc,get_sc,get_sc_feature,get_dataloader, get_fc_edgeWeight, get_sc_dataloader, get_fc_dataloader
from torch_geometric.utils import dense_to_sparse
from torch_geometric.data import Data
from scipy import sparse as sp
import hashlib

os.environ['KMP_DUPLICATE_LIB_OK'] = 'TRUE'

def decide_dataset(root):  # 返回标签
    if 'ABIDE' in root:
        class_dict = {
            "HC": 0,
            "ASD": 1,
        }
    elif 'Multi' in root or "xinxiang" in root or "zhongda" or "multi_SCFC" in root:
        class_dict = {
            "HC": 0,
            "MDD": 1,
        }
    elif "HCP" in root:
        class_dict = {
            "female": 0,
            "male": 1,
        }
    return class_dict

def read_dataset_fc_regionSeries(root, label_files, files):  # 返回时间序列
    FC_dir = "RegionSeries.mat"
    if 'ABIDE' in root:
        subj_fc_dir = os.path.join(root, label_files, files)
        subj_mat_fc = np.loadtxt(subj_fc_dir)[:176, :90]
    elif 'Multi' in root:
        subj_fc_dir = os.path.join(root, label_files, files)
        subj_mat_fc = io.loadmat(subj_fc_dir)['ROISignals_AAL'][:170, :]
    else:  # zhonda、xinxiang、ASD的读法都一样
        subj_fc_dir = os.path.join(root, label_files, files, FC_dir)
        subj_mat_fc = io.loadmat(subj_fc_dir)['RegionSeries']
    return subj_mat_fc

def read_dataset_fc_edgeWeight(root, label_files, files):  # 返回时间序列
    FC_dir = "EdgeWeight.mat"
    subj_fc_dir = os.path.join(root, label_files, files, FC_dir)
    subj_mat_fc = io.loadmat(subj_fc_dir)['EdgeWeight']
    return subj_mat_fc

def read_dataset_fc_region_features(root, label_files, files):  # 返回时间序列
    FC_dir = "region_features_norm.mat"
    subj_fc_dir = os.path.join(root, label_files, files, FC_dir)
    subj_mat_fc = io.loadmat(subj_fc_dir)['region_features']
    return subj_mat_fc

def read_dataset_sc_connectivity(root, label_files, files):  # 返回sc connectivity
    SC_connectivity_dir = "dti_FACT_45_02_1_1_Matrix_FA_AAL_Contract_90_2MM_90.txt"
    subj_sc_connectivity_dir = os.path.join(root, label_files, files, SC_connectivity_dir)
    subj_sc_connectivity = np.loadtxt(subj_sc_connectivity_dir)
    # subj_sc_connectivity = io.loadmat(subj_sc_connectivity_dir)['connectivity']
    # subj_sc_connectivity = subj_sc_connectivity[0:90, 0:90]
    return subj_sc_connectivity

def read_dataset_sc_features(root, label_files, files):  # 返回sc特征
    SC_features_dir = "region_features_norm.mat"
    subj_sc_feature_dir = os.path.join(root, label_files, files, SC_features_dir)
    subj_sc_feature = io.loadmat(subj_sc_feature_dir)['region_features']
    return subj_sc_feature

def read_dataset_fc_region_features(root, label_files, files):  # 返回时间序列
    FC_dir = "region_features_norm.mat"
    subj_fc_dir = os.path.join(root, label_files, files, FC_dir)
    subj_mat_fc = io.loadmat(subj_fc_dir)['region_features']
    return subj_mat_fc

def d_constraint(X, D, a):
    n, d = X.shape
    sum_dist = 0

    for i in range(n):
        for j in range(i+1, n):
            if D[i, j] == 1:
                d_ij = X[i] - X[j]
                dist_ij = distance1(a, d_ij)
                sum_dist += dist_ij
    fD = gf(sum_dist)
    return fD

def gf(sum_dist):
    fD = np.log(sum_dist)
    return fD

def distance1(a, d_ij):
    fudge = 0.000001
    dist_ij = np.sqrt(np.dot(d_ij**2, a))  # distance between X[i] and X[j]
    return dist_ij

class FSDataset:
    def __init__(self, args):
        self.sc = []
        self.fc = []
        self.scfc = []
        self.class_dict = decide_dataset(args.path)
        self.y = []

        root = args.path
        label_list = os.listdir(root)
        label_list.sort()

        for label_files in label_list:
            list = os.listdir(os.path.join(root, label_files))
            list.sort()
            label = torch.LongTensor([self.class_dict[label_files]])
            for files in list:
                ############
                sc_connectivity = read_dataset_sc_connectivity(root, label_files, files)
                sc_feature = read_dataset_sc_connectivity(root, label_files, files)
                subj_sc_adj = get_sc(sc_connectivity, args)  # 构造邻接矩阵
                subj_sc_feature = get_sc_feature(sc_feature, args)  # 构造特征
                # 构造sc Data
                # sc_data = get_sc_dataloader(subj_sc_adj, subj_sc_feature)
                # sc_data.y = label
                # self.sc.append(sc_data)
                ############
                fc_timeseries = read_dataset_fc_regionSeries(root, label_files, files)  # 这里得到的已经是节点数*长度的时间序列
                subj_fc_adj = get_Pearson_fc(fc_timeseries,
                                                   args)
                # fc_edgeWeight = read_dataset_fc_edgeWeight(root, label_files, files)
                d = np.ones((90, 90)) - np.tril(np.ones((90, 90)))
                a = np.ones(90)
                re = d_constraint( subj_fc_adj, d, a)
                re = math.exp(-(re/2*args.bandwidth*args.bandwidth))
                # subj_fc_adj = get_Pearson_fc(fc_timeseries,
                #                                    args)  # 这里传参是时间序列，返回的是邻接矩阵，以方法命名，因为可能有不同的构造方法，例如度矩阵等construct_Degree_fc;threshold等参数通过args传递
                fc_region_features = read_dataset_fc_region_features(root, label_files, files)
                subj_fc_feature = re * get_fc_edgeWeight(subj_fc_adj ,
                                                        args)  # 同样传参时间序列，返回特征，注意这里特征和上面邻接矩阵都返回的是矩阵，构造Data放在后面，以防有的会直接使用这两个矩阵，可以将Data构造部分注释掉
                # subj_fc_feature = re * subj_fc_adj
                # 构造fc Data
                # fc_data = get_dataloader(subj_fc_adj, subj_fc_feature)
                # fc_data.y = label
                # self.fc.append(fc_data)
                ############
                # 构造sc fc Data

                # self.fuse_weight_1 = torch.nn.Parameter(torch.FloatTensor(1), requires_grad=True)
                # self.fuse_weight_2 = torch.nn.Parameter(torch.FloatTensor(1), requires_grad=True)
                # self.fuse_weight_1.data.fill_(1)
                # self.fuse_weight_2.data.fill_(1)
                feature = np.identity(90) +  subj_fc_feature + subj_sc_feature
                adj = np.identity(90)

                data = get_dataloader(adj, feature)
                data.y = label
                # data.fc_x = get_dataloader(subj_fc_adj, subj_fc_feature).x
                # data.fc_edge_index = get_dataloader(subj_fc_adj, subj_fc_feature).edge_index
                self.scfc.append(data)
                self.y.append(label)

        self.choose_data = self.scfc


import argparse
import matplotlib.pyplot as plt
from matplotlib.pyplot import MultipleLocator
import numpy as np
from collections.abc import Iterable

color_tuple = ('#7f7f7f', '#2ca02c', '#e377c2', '#1f77b4')

color_tuple_targeted = ('#7f7f7f', '#2ca02c', '#e377c2', '#9467bd', '#8c564b','#1f77b4')

style_tuple = ('-', '-', '-', '-', '-', '-', '-', '-')
marker_tuple = ('s', '^', '^', '^', '^', '*', '*', 'h')
ms_tuple = (8, 9, 9, 9, 9, 9, 9, 9)

def plot_curve(curve_arr, curve_label, title='', color=None, style=None, curve_type='pr', targeted=False):
    filter_label = ('Original',) if targeted else ('',)
    if not isinstance(curve_arr, Iterable):
        curve_arr = (curve_arr,)
        curve_label = (curve_label,)

    if color is None:
        color = color_tuple_targeted if targeted else color_tuple

    if style is None:
        style = style_tuple

    fig, ax = plt.subplots()
    plt.figure(1)

    for i, curve in enumerate(curve_arr):
        if curve_label[i] in filter_label:
            continue
        x = curve[:, 0]
        y = curve[:, 1]
        plt.plot(x, y, label=curve_label[i], c=color[i], ls=style[i], lw=2.5)

    x_major_locator = MultipleLocator(0.2 if curve_type == 'pr' else 200)
    y_major_locator = MultipleLocator(0.1)
    loc = (0.05, 0.05) if targeted else (0.05, 0.65)
    ax.xaxis.set_major_locator(x_major_locator)
    ax.yaxis.set_major_locator(y_major_locator)
    plt.ylim(0.05, 0.7)
    plt.xlabel('Recall' if curve_type == 'pr' else 'Number of top ranked samples')
    plt.ylabel('Precision')
    plt.title(title)
    plt.legend(loc=loc, framealpha=0.7)
    # plt.grid(linestyle='--')
    fig.subplots_adjust(left=0.09, right=0.99)
    fig.set_size_inches(6, 5.5)
    filename = curve_type + ('_targeted' if targeted else '')
    plt.savefig('../documents/{}-{}.pdf'.format(title, filename), dpi=600, format='pdf', transparent=True)
    # plt.show()

def plot_ablation(row, para_type='lambda'):
    import pandas as pd
    data = pd.DataFrame(pd.read_excel('../documents/AttackMAP.xlsx', sheet_name=0))
    curve_label = ['Original', 'HAG', 'SDHA', 'Ours']

    parameters = []
    map_value = []
    for i in range(row[0], row[1]):
        value = data.loc[i].values
        if i == row[0]:
            parameters = [float(v) for v in value[1:7]]
        else:
            map_value.append([float(v) for v in value[1:7]])
    curve_arr = np.array([list(zip(parameters, val)) for val in map_value])

    color, style = color_tuple, style_tuple
    fig, ax = plt.subplots()
    plt.figure(1)

    for i, curve in enumerate(curve_arr):
        x = range(len(curve[:, 0]))
        y = curve[:, 1] * 100
        plt.plot(x, y, label=curve_label[i], c=color[i], ls=style[i], lw=2.5, markerfacecolor='none'
                 , markeredgewidth=2, marker=marker_tuple[i], markersize=ms_tuple[i])
    plt.xticks(range(curve_arr.shape[1]), labels=[str(i) for i in curve_arr[0, :, 0]])
    plt.xlabel('weighting factor ' + ('$\lambda$' if para_type == 'lambda' else '$\mu$'))
    plt.ylabel('MAP(%)')
    plt.legend(loc=(0.05, 0.65))
    plt.grid(linestyle='--')
    fig.subplots_adjust(left=0.1, right=0.99)
    plt.savefig('../documents/{}-{}.pdf'.format('ablation', para_type), dpi=600, format='pdf', transparent=True)
    # plt.show()

def parser_arguments():
    parser = argparse.ArgumentParser()
    parser.add_argument('--dataset_name', dest='dataset', default='NUS-WIDE',
                        choices=['CIFAR-10', 'ImageNet', 'FLICKR-25K', 'NUS-WIDE', 'MS-COCO'],
                        help='name of the dataset')
    parser.add_argument('--hash_method', dest='hash_method', default='DPH',
                        choices=['DPH', 'DPSH', 'HashNet', 'CSQ'],
                        help='deep hashing methods')
    parser.add_argument('--backbone', dest='backbone', default='AlexNet',
                        choices=['AlexNet', 'VGG11', 'VGG16', 'VGG19', 'ResNet18', 'ResNet50'],
                        help='backbone network')
    parser.add_argument('--code_length', dest='bit', type=int, default=32, help='length of the hashing code')
    parser.add_argument('--type', dest='type', type=str, default='pr', help='curve type')
    parser.add_argument('--targeted', dest='targeted', action="store_true", default=False, help='targeted attack')
    return parser.parse_args()

if __name__ == '__main__':
    args = parser_arguments()
    attack_model = '{}_{}_{}_{}'.format(args.dataset, args.hash_method, args.backbone, args.bit)
    filename = args.type + ('_targeted' if args.targeted else '')
    arr = np.load('../log/{}/{}.npy'.format(attack_model, filename))
    label = np.loadtxt('../log/{}/{}.txt'.format(attack_model, filename), dtype=str)
    label = [l if 'Ours' not in l else 'Ours' for l in label]
    plot_curve(arr, label, title=args.dataset, curve_type=args.type, targeted=args.targeted)
    # plot_ablation(row=(89, 94), para_type='lambda')
    # plot_ablation(row=(97, 102), para_type='mu')


import re
import argparse

# parse input arguments
parser = argparse.ArgumentParser(description="""Search and replace Freescale pre-processor
                                             macros for memory access with uvisor ones""",
                                 usage='%(prog)s filename(s)')
parser.add_argument('filenames', nargs='+', help='input file(s) to process', metavar='FILE')
parser.parse_args()

# regex array
regex_dict = []

# regular read, 1/2 args, 8/16/32 bits
regex_dict.append({
    'src': r'(#define\s+HW_[A-Za-z_0-9]+_RD\((.+)\)\s+)\((HW_[A-Za-z_0-9]+)\(.+\).U\)',
    'dst': lambda p: p.group(1) + '(' + 'ADDRESS_READ('+p.group(3).lower()+'_t, '+p.group(3)+'_ADDR('+p.group(2)+'))' + ')'
})

# regular write, 2/3 args, 8/16/32 bits
regex_dict.append({
    'src': r'(#define\s+HW_[A-Za-z_0-9]+_WR\((.+), v\)\s+)\((HW_[A-Za-z_0-9]+)\(.+\).U = \(v\)\)',
    'dst': lambda p: p.group(1) + '(' + 'ADDRESS_WRITE('+p.group(3).lower()+'_t, '+p.group(3)+'_ADDR('+p.group(2)+'), v)' + ')'
})

# bitband read, 1/2 args, 8/16/32 bits
regex_dict.append({
    'src': r'(#define\s+BR_[A-Za-z_0-9]+\((.+)\)\s+)\(BITBAND_ACCESS(8|16|32)\((.+)\)\)',
    'dst': lambda p: p.group(1) + '(' + 'ADDRESS_READ(uint'+p.group(3)+'_t, '+'BITBAND_ADDRESS'+p.group(3)+'('+p.group(4)+'))' + ')'
})

# bitband write, 2/3 args, 8/16/32 bits
regex_dict.append({
    'src': r'(#define\s+BW_[A-Za-z_0-9]+\((.+) v\)\s+)\(BITBAND_ACCESS(8|16|32)\((.+)\) = \(v\)\)',
    'dst': lambda p: p.group(1) + '(' + 'ADDRESS_WRITE(uint'+p.group(3)+'_t, '+'BITBAND_ADDRESS'+p.group(3)+'('+p.group(4)+'), v)' + ')'
})

# regular read, 2/3 args, 8/16/32 bits
regex_dict.append({
    'src': r'(#define\s+BR_[A-Za-z_0-9]+\((.+)\)\s+)\((HW_[A-Za-z_0-9]+)\(.+\)\.B\.(.+)\)',
    'dst': lambda p: p.group(1) + '(' + 'UNION_READ('+p.group(3).lower()+'_t, '+p.group(3)+'_ADDR('+p.group(2)+'), U, B.'+p.group(4)+')' + ')'
})

# iterate over all lines in given input files
# no backup file is create since it is exepcted to run the script within git
lines_changed = 0
for line in fileinput.input(inplace=1):
    # iterate over all regular expressions (stop at the first one matching)
    for rgx in regex_dict:
        [line, n] = re.subn(rgx['src'], rgx['dst'], line.rstrip())
        if n > 0:
            lines_changed = lines_changed + 1
            break

    # write line to file (fileinput automatically redirects output to file)
    print(line)

# print quick report
if(lines_changed == 1):
    print('Changed ' + str(lines_changed) + ' line')
else:
    print('Changed ' + str(lines_changed) + ' lines')


import numpy as np
from sklearn.linear_model import LinearRegression
from sklearn.model_selection import KFold, GridSearchCV

from ._utils_boot import boot_manual, draw_weights

def fit_nuisance_pliv_partial_x(y, x, d, z, ml_m, ml_g, ml_r, smpls, g_params=None, m_params=None, r_params=None):
    assert z.ndim == 2
    g_hat = []
    for idx, (train_index, test_index) in enumerate(smpls):
        if g_params is not None:
            ml_g.set_params(**g_params[idx])
        g_hat.append(ml_g.fit(x[train_index], y[train_index]).predict(x[test_index]))

    m_hat = []
    m_hat_array = np.zeros_like(z, dtype='float64')
    for i_instr in range(z.shape[1]):
        this_instr_m_hat = []
        for idx, (train_index, test_index) in enumerate(smpls):
            if m_params is not None:
                ml_m.set_params(**m_params[i_instr][idx])
            this_instr_m_hat.append(ml_m.fit(x[train_index], z[train_index, i_instr]).predict(x[test_index]))
            m_hat_array[test_index, i_instr] = this_instr_m_hat[idx]
        m_hat.append(this_instr_m_hat)

    r_hat = []
    r_hat_array = np.zeros_like(d, dtype='float64')
    for idx, (train_index, test_index) in enumerate(smpls):
        if r_params is not None:
            ml_r.set_params(**r_params[idx])
        r_hat.append(ml_r.fit(x[train_index], d[train_index]).predict(x[test_index]))
        r_hat_array[test_index] = r_hat[idx]

    r_hat_tilde = LinearRegression(fit_intercept=True).fit(z - m_hat_array, d - r_hat_array).predict(z - m_hat_array)

    return g_hat, r_hat, r_hat_tilde

def tune_nuisance_pliv_partial_x(y, x, d, z, ml_m, ml_g, ml_r, smpls, n_folds_tune, param_grid_g, param_grid_m, param_grid_r):
    g_tune_res = [None] * len(smpls)
    for idx, (train_index, _) in enumerate(smpls):
        g_tune_resampling = KFold(n_splits=n_folds_tune, shuffle=True)
        g_grid_search = GridSearchCV(ml_g, param_grid_g,
                                     cv=g_tune_resampling)
        g_tune_res[idx] = g_grid_search.fit(x[train_index, :], y[train_index])

    m_tune_res = [[None] * len(smpls) for i in range(z.shape[1])]
    for i_instr in range(z.shape[1]):
        for idx, (train_index, _) in enumerate(smpls):
            m_tune_resampling = KFold(n_splits=n_folds_tune, shuffle=True)
            m_grid_search = GridSearchCV(ml_m, param_grid_m,
                                         cv=m_tune_resampling)
            m_tune_res[i_instr][idx] = m_grid_search.fit(x[train_index, :], z[train_index, i_instr])

    r_tune_res = [None] * len(smpls)
    for idx, (train_index, _) in enumerate(smpls):
        r_tune_resampling = KFold(n_splits=n_folds_tune, shuffle=True)
        r_grid_search = GridSearchCV(ml_r, param_grid_r,
                                     cv=r_tune_resampling)
        r_tune_res[idx] = r_grid_search.fit(x[train_index, :], d[train_index])

    g_best_params = [xx.best_params_ for xx in g_tune_res]
    m_best_params = [[xx.best_params_ for xx in m_tune_res[i_instr]] for i_instr in range(z.shape[1])]
    r_best_params = [xx.best_params_ for xx in r_tune_res]

    return g_best_params, m_best_params, r_best_params

def pliv_partial_x_dml1(y, x, d, z, g_hat, r_hat, r_hat_tilde, smpls, score):
    thetas = np.zeros(len(smpls))
    n_obs = len(y)

    for idx, (_, test_index) in enumerate(smpls):
        u_hat = y[test_index] - g_hat[idx]
        w_hat = d[test_index] - r_hat[idx]
        thetas[idx] = pliv_partial_x_orth(u_hat, w_hat, r_hat_tilde[test_index], d[test_index], score)
    theta_hat = np.mean(thetas)

    u_hat = np.zeros_like(y, dtype='float64')
    w_hat = np.zeros_like(d, dtype='float64')
    for idx, (_, test_index) in enumerate(smpls):
        u_hat[test_index] = y[test_index] - g_hat[idx]
        w_hat[test_index] = d[test_index] - r_hat[idx]
    se = np.sqrt(var_pliv_partial_x(theta_hat, d, u_hat, w_hat, r_hat_tilde, score, n_obs))

    return theta_hat, se

def pliv_partial_x_dml2(y, x, d, z, g_hat, r_hat, r_hat_tilde, smpls, score):
    n_obs = len(y)
    u_hat = np.zeros_like(y, dtype='float64')
    w_hat = np.zeros_like(d, dtype='float64')
    for idx, (_, test_index) in enumerate(smpls):
        u_hat[test_index] = y[test_index] - g_hat[idx]
        w_hat[test_index] = d[test_index] - r_hat[idx]
    theta_hat = pliv_partial_x_orth(u_hat, w_hat, r_hat_tilde, d, score)
    se = np.sqrt(var_pliv_partial_x(theta_hat, d, u_hat, w_hat, r_hat_tilde, score, n_obs))

    return theta_hat, se

def var_pliv_partial_x(theta, d, u_hat, w_hat, r_hat_tilde, score, n_obs):
    assert score == 'partialling out'
    var = 1/n_obs * 1/np.power(np.mean(np.multiply(r_hat_tilde, w_hat)), 2) * \
        np.mean(np.power(np.multiply(u_hat - w_hat*theta, r_hat_tilde), 2))

    return var

def pliv_partial_x_orth(u_hat, w_hat, r_hat_tilde, d, score):
    assert score == 'partialling out'
    res = np.mean(np.multiply(r_hat_tilde, u_hat))/np.mean(np.multiply(r_hat_tilde, w_hat))

    return res

def boot_pliv_partial_x(theta, y, d, z, g_hat, r_hat, r_hat_tilde,
                        smpls, score, se, bootstrap, n_rep):
    n_obs = len(y)
    weights = draw_weights(bootstrap, n_rep, n_obs)
    assert np.isscalar(theta)
    boot_theta, boot_t_stat = boot_pliv_partial_x_single_treat(theta, y, d, z, g_hat, r_hat, r_hat_tilde,
                                                               smpls, score, se, weights, n_rep)
    return boot_theta, boot_t_stat

def boot_pliv_partial_x_single_treat(theta, y, d, z, g_hat, r_hat, r_hat_tilde,
                                     smpls, score, se, weights, n_rep):
    assert score == 'partialling out'
    u_hat = np.zeros_like(y, dtype='float64')
    w_hat = np.zeros_like(d, dtype='float64')
    for idx, (_, test_index) in enumerate(smpls):
        u_hat[test_index] = y[test_index] - g_hat[idx]
        w_hat[test_index] = d[test_index] - r_hat[idx]

    J = np.mean(-np.multiply(r_hat_tilde, w_hat))

    psi = np.multiply(u_hat - w_hat*theta, r_hat_tilde)

    boot_theta, boot_t_stat = boot_manual(psi, J, smpls, se, weights, n_rep)

    return boot_theta, boot_t_stat


'''
Author: Jeffrey Sultan
Date: 8/4/22
Dataset: Uber Fares Data csv
App URL: ###############
Description: This interactive application containing Uber ride data from 2009 to 2015.
            The data is depicted in a series of visualizations to illustrate various metrics that
            were found. The original data table was updated to remove unneeded information. Date,
            time and straight-line calculated distance between pickup and drop-off location was
            also added to the data table.

'''
import streamlit as st
import pandas as pd
import numpy as np
import plotly.express as px
import geopy.distance
import seaborn as sns
import matplotlib.pyplot as plt

# Set constants
ARRIVAL_DATE_COLUMN = 'pickup_datetime'
DATA = "UberData.csv"
#---------------------------------------------------------------------
# Code checklist: Function: function w/ defualt parameter, two parameters, returns value
def getData(key, dict='m'):
    monthDict = {1:"January",2:"February",3:"March",4:"April",5:"May",6:"June",7:"July",8:"August",9:"September",10:"October",11:"November",12:"Decemeber"}
    weekTpl = ("Sunday","Monday","Tuesday","Wednesday","Thursday","Friday","Saturday")

    if dict == 'm':
        return monthDict[key]
    elif dict == 'w':
        return weekTpl[key]
#---------------------------------------------------------------------
# Set page layout and create header
st.set_page_config(layout='wide')

# Initialize application with title, header and descriptive information
# st cols(#1, #2) where #1 = width of col1 and #2 = width of col2
row0_1, row0_2 = st.columns((3,1))
with row0_1:
    st.title('Uber pickups in NYC')
with row0_2:
    st.text("")
    st.subheader('Streamlit App created by:')
    st.text("Jeffrey Sultan")
# col1 spacer = 0.1 padding, col2 width, col3 spacer = 0.1 padding
row3_spacer1, row3_1, row3_spacer2 = st.columns((0.1, 1, 0.1))
with row3_1:
    st.markdown("This interactive application containing Uber ride data from 2009 to 2015. The data is depicted in a series of visualizations to illustrate various metrics that were found. ")
    st.markdown("The original data table was updated to remove unneeded information. ")
    st.markdown("Date, time and straight-line calculated distance between pickup and drop-off location was also added to the data table. ")
st.title('')

# creating a sidebar to display additional project information
# Code checklist - At least one function that does not return a value
def load_sidebar():
    st.sidebar.title("About")
    st.sidebar.info(
        """
        This application uses the following charts to visualize the data collected:
        \n1) A map of the pickup location
        \n2) A scatter plot of fare vs distance
        \n3) A pie chart of the date/time distribution
        \n4) A line chart of the pickup time distribution
        \n5) A scatter plot with linear prediction
        """
    )
    st.sidebar.title("Author - Date:")
    st.sidebar.info("Author: Jeffrey Sultan")
    st.sidebar.info("Date: 04Aug2022")
    st.sidebar.info("Dataset: Uber Fares Data CSV")
load_sidebar()
#------------------------------------------------------------------------
# Section 0 - Start with loading data from the CSV

def load_data():
    data = pd.read_csv(DATA)
    lowercase = lambda x: str(x).lower()
    data.rename(lowercase, axis='columns', inplace=True)

    return data

# Loading bar used to work but for some reason i no longer see it
data_load_state = st.text('Loading data...')
data = load_data()
data_load_state.text("")
#------------------------------------------------------------------------
# Section 1 - Reframe the data provided to be more accessible and removing outliers

# The line below reframes the pickup and dropoff locations:
# Latitude should be 40 to 41. Longitude should be -73 to -75
data = data[(data.pickup_latitude<41) & (data.dropoff_latitude<41) &
        (data.pickup_latitude>40) & (data.dropoff_latitude>40) &
        (data.pickup_longitude<-73) & (data.dropoff_longitude<-73) &
        (data.pickup_longitude>-75) & (data.dropoff_longitude>-75)]

# Renaming the pickup latitude and longitude to work with map utility
data = data.rename({'pickup_longitude': 'longitude', 'pickup_latitude': 'latitude'}, axis=1)
data[ARRIVAL_DATE_COLUMN] = pd.to_datetime(data[ARRIVAL_DATE_COLUMN])

# Remove unnecessary columns (Unnamed and key columns)
# Code checklist: dataframe: drop columns
data.drop(['unnamed: 0', 'key'], axis=1, inplace=True)

# Create new columns to split the arrival date column from date/time conversion
# Code checklist: dataframe: add columns
data['year'] = data.pickup_datetime.dt.year
data['month'] = data.pickup_datetime.dt.month
data['weekday'] = data.pickup_datetime.dt.weekday
data['hour'] = data.pickup_datetime.dt.hour

# Using the geopy module, we can create a new column that evaluates the distance between pickup and drop off locations.
# Note that the pickup lat/long columns were renamed above from 'pickup_latitude' to 'latitude' and same for longitude
# geopy.distance.distance uses the geodesic distance which is the shortest distance on the surface of an ellipsoidal model of the earth (Karney 2013)
# The calculated distance is in meters, rounded to 2 decimal places and placed in a new column titled 'distance'
data['distance']=[round(geopy.distance.distance((data.latitude[i], data.longitude[i]),(data.dropoff_latitude[i], data.dropoff_longitude[i])).m,2) for i in data.index]

# Reframe the data set to exclude fare_amount outliers more than $100 and distance outliers less than 1000 meters
data = data[(data.fare_amount<=100)]
data = data[(data.distance > 1000)]

#------------------------------------------------------------------------
# Section 2 - Option to show the reframed data from above + stats

# Checkbox to show raw data
# Code checklist: pandas: sort column asc/desc
if st.checkbox('Show data table'):
    st.subheader('Reframed data with added columns')
    fare_median = data['fare_amount'].median()
    dist_median = data['distance'].median()
    st.text(f'{len(data)} Entries\t ${fare_median} Median of Ride Fares\t {dist_median}m Median of Distances Travelled')
    st.write(data.sort_values('distance', ascending=False))
#------------------------------------------------------------------------
# Section 3 - Visualize the pickup locations by hour on a map of NYC
# Separation bar html copied from streamlit discussion bar - see references
st.markdown("""
 """, unsafe_allow_html=True)
st.subheader('| VISUALIZING MAP OF ALL PICKUP LOCATIONS DEPENDENT ON MONTH OR HOUR')
map_option = st.radio(
     "How to filter the data appearing on the map?",
     ('Pickup Hour', 'Pickup Month', 'Pickup Weekday'))

# Code checklist: Data structures: Dictionary & Streamlit UI Controls: Slider
# Code checklist: At least one function that is called twice
if map_option == 'Pickup Month':
    # Sliding bar for mapping feature: min = (1-Jan), max = (12-Dec)
    month_to_filter = st.slider('MONTH OF THE YEAR', 1, 12)
    selected_month = getDict(month_to_filter, 'm')
    filtered_data = data[data[ARRIVAL_DATE_COLUMN].dt.month == month_to_filter]
    st.subheader(f"Map of all pickups in {selected_month}")
    st.text(f"Showing {len(filtered_data)} pickups")
    st.map(filtered_data)
elif map_option == 'Pickup Weekday':
    # Sliding bar for mapping feature: min = 0hr, max = 23hr, default = 17hr
    weekday_to_filter = st.slider('DAY OF WEEK', 0, 6)
    selected_weekday = getData(weekday_to_filter, 'w')
    filtered_data = data[data[ARRIVAL_DATE_COLUMN].dt.weekday == weekday_to_filter]
    st.subheader(f'Map of all pickups on {selected_weekday}')
    st.text(f"Showing {len(filtered_data)} pickups")
    st.map(filtered_data)
elif map_option == 'Pickup Hour':
    # Sliding bar for mapping feature: min = 0hr, max = 23hr, default = 17hr
    hour_to_filter = st.slider('HOUR OF THE DAY', 0, 23, 17)
    filtered_data = data[data[ARRIVAL_DATE_COLUMN].dt.hour == hour_to_filter]
    st.subheader('Map of all pickups at %s:00' % hour_to_filter)
    st.text(f"Showing {len(filtered_data)} pickups")
    st.map(filtered_data)

#------------------------------------------------------------------------
# Section 4 - Scatter Plot visualization of fares vs distance with optional filters
st.markdown("""
 """, unsafe_allow_html=True)
st.subheader('| VISUALIZING SCATTERPLOT OF FARES VS DISTANCE AND VARIABLE HUE')

fig4_option = st.selectbox('USE VARIABLE TO SET HUE',('month', 'year', 'weekday', 'hour'))
fig4 = plt.figure()
# next line somehow controls grid lines in my graph. Copied from a seaborn discussion on stack exchange. see refs
sns.set(rc={'figure.figsize':(8,6)})
plt.ylim(0, 100)

plt.title('Fares vs Distance')
plt.xlabel('Straight-line Distance (meters)')
plt.ylabel('Fare ($)')

# Scatter plot created using the seaborn module
sns.scatterplot(data=data, x='distance',y='fare_amount',hue=fig4_option, s=10)
st.pyplot(fig4)

# Next create a pie chart to illustrate the percentage of entries found for each hue option selected above
if fig4_option == 'year':
    ind1 = pd.DataFrame(data.groupby('year').year.count()).rename(columns={'year': 'count'}).reset_index()
    g1 = px.pie(ind1,
                values='count',
                names='year',
                color='year',
                color_discrete_map={'2009': 'red', '2010': 'blue', '2011': 'green', '2012': 'yellow', '2013': 'orange',
                                    '2014': 'purple', '2015': 'brown'},
                title='| PERCENTAGE OF DATA RECORDED FROM EACH YEAR')
    g1.update_traces(textposition='inside',
                     textinfo='percent+label',
                     showlegend=False)
    st.plotly_chart(g1, use_container_width=True)
elif fig4_option == 'month':
    ind1 = pd.DataFrame(data.groupby('month').month.count()).rename(columns={'month': 'count'}).reset_index()
    g1 = px.pie(ind1,
                values='count',
                names='month',
                color='month',
                title='| PERCENTAGE OF DATA RECORDED FROM EACH MONTH')
    g1.update_traces(textposition='inside',
                     textinfo='percent+label',
                     showlegend=False)
    st.plotly_chart(g1, use_container_width=True)
elif fig4_option == 'weekday':
    ind1 = pd.DataFrame(data.groupby('weekday').weekday.count()).rename(columns={'weekday': 'count'}).reset_index()
    g1 = px.pie(ind1,
                values='count',
                names='weekday',
                color='weekday',
                title='| PERCENTAGE OF DATA RECORDED FROM EACH WEEKDAY')
    g1.update_traces(textposition='inside',
                     textinfo='percent+label',
                     showlegend=False)
    st.plotly_chart(g1, use_container_width=True)
elif fig4_option == 'hour':
    ind1 = pd.DataFrame(data.groupby('hour').hour.count()).rename(columns={'hour': 'count'}).reset_index()
    g1 = px.pie(ind1,
                values='count',
                names='hour',
                color='hour',
                title='| PERCENTAGE OF DATA RECORDED FROM EACH HOUR')
    g1.update_traces(textposition='inside',
                     textinfo='percent+label',
                     showlegend=False)
    st.plotly_chart(g1, use_container_width=True)

#------------------------------------------------------------------------
# Section 5 - Chart representing the data from each year and time of day
st.markdown("""
 """, unsafe_allow_html=True)
st.subheader('| VISUALIZING DATA OF RELATED TO DATE & TIME INFORMATION')

# Bar chart that illustrates the amount of data recorded from each time of day
fig5_2_option = st.selectbox('USE VARIABLE TO FILTER DATA', ('year', 'month'))
if fig5_2_option == 'year':
    fig5_year_to_filter = st.slider('YEAR', 2009, 2015)
    filtered_data = data[data[ARRIVAL_DATE_COLUMN].dt.year == fig5_year_to_filter]
elif fig5_2_option == 'month':
    fig5_month_to_filter = st.slider('MONTH', 1, 12)
    filtered_data = data[data[ARRIVAL_DATE_COLUMN].dt.month == fig5_month_to_filter]
ind2 = pd.DataFrame(filtered_data.groupby('hour').hour.count()).rename(columns={'hour':'count'}).reset_index()
g3 = px.line(ind2,
            x='hour',
            y='count',
            title='| POPULAR PICKUP TIMES EACH DAY')
st.plotly_chart(g3, use_container_width=True)
#------------------------------------------------------------------------
# Section 6 - Scatter plot of distance vs fare amount and prediction of fare based on distance
st.markdown("""
 """, unsafe_allow_html=True)
st.subheader('| LINEAR REGRESSION OF DISTANCE VS FARE AMOUNT')

# Manual calculation of Y-estimate using Ordinary Least Squares (could not get stats package to work correctly)
x = filtered_data['distance']
y = filtered_data['fare_amount']

# Calculate the mean of X and y
xmean = np.mean(x)
ymean = np.mean(y)

# Calculate the terms needed for the numerator and denominator of beta
df_temp = pd.DataFrame()
df_temp['xycov'] = (x - xmean) * (y - ymean)
df_temp['xvar'] = (y - xmean)**2

# Calculate beta and alpha
beta = df_temp['xycov'].sum() / df_temp['xvar'].sum()
alpha = ymean - (beta * xmean)

# Integrate user interaction to predict fare amount vs distance
selected_distance = st.slider("DISTANCE IN METERS", 0, 50000, 10000)
y_prediction_from_selected = alpha + beta * selected_distance
st.write('Estimated Fare for Selected Distance = \\$', round(y_prediction_from_selected, 2))

# Plot manual calculation against actual data
ypred = alpha + beta * x
fig7 = plt.figure(figsize=(12, 6))
plt.plot(x, ypred)     # regression line
plt.plot(x, y, 'ro')   # scatter plot showing actual data
plt.plot(selected_distance, y_prediction_from_selected, marker="o", markersize=15, markerfacecolor="blue")
plt.title('Linear Regression Based on Distance')
plt.xlabel('Distance (meters)')
plt.ylabel('Fare Amount ($)')

# Plot the regression line against actual distance vs fare data
st.pyplot(fig7)
#-----------------------------------------------------------------------------------
st.markdown("""
 """, unsafe_allow_html=True)
st.subheader('| ONLINE EXAMPLES REFERENCED')

st.write("https://towardsdatascience.com/introduction-to-linear-regression-in-python-c12a072bedf0")
st.write("https://www.kaggle.com/code/yasserh/uber-fare-prediction-comparing-best-ml-models")
st.write("https://pythonwife.com/pie-chart-with-plotly/")
st.write("https://seaborn.pydata.org/tutorial/regression.html")
st.write("https://github.com/tdenzl/BuLiAn")
st.write("https://discuss.streamlit.io/t/horizontal-separator-line/11788")
st.write("https://stackoverflow.com/questions/31594549/how-to-change-the-figure-size-of-a-seaborn-axes-or-figure-level-plot")


from typing import List
from datetime import datetime
from dateutil.relativedelta import relativedelta
from fastapi import HTTPException
from sqlalchemy import desc, func
from db import models
from sqlalchemy.orm.session import Session
from schemas.api import RankingUser, Book, Report, TotalInfo, User as UserSchema

def get_books_order_by_query(db: Session, user_id: str, order_by: str) -> List[Book]:

  user_orm = db.query(models.User).filter(models.User.user_id == user_id).first()

  if user_orm is None:
    raise HTTPException(
            status_code=404,
            detail="Not Found User"
        )
  user = UserSchema.from_orm_custom(user_orm)
  books = user.books
  if(order_by == "title"):
    books = sorted(books, key=lambda x:x.title)

  if(order_by == "author"):
    books = sorted(books, key=lambda x: x.author)

  if(order_by == "published_date"):
    books = sorted(books, key=lambda x: x.published_date)

  if(order_by == "created_at"):
    books = sorted(books, key=lambda x: x.created_at)

  return books

def create_report(db: Session, user_id: str, type: str) -> Report:
  if not type in ['quantity', 'page']:
    raise HTTPException(status_code=400, detail='invalid query')
  now = datetime.now()
  months_name = list(range(12))
  labels = months_name[now.month:]+months_name[:now.month]
  print(labels)
  months = []
  data_list = []
  for i in range(12):
    previous = now - relativedelta(months=11-i)
    read_pages = db.query(models.ReadPage).filter(models.ReadPage.user_id == user_id, models.ReadPage.year == previous.year, models.ReadPage.month == previous.month).all()
    if len(read_pages) == 0 and len(months) == 0:
      continue
    data = 0
    if type == 'page':
      for read_page in read_pages:
        data += read_page.pages
    elif type == 'quantity':
      for read_page in read_pages:
        data += read_page.quantity
    data_list.append(data)
    months.append(labels[i]+1)
  report = Report(
    months=months,
    data=data_list
  )
  return report

def get_ranking(db: Session, type: str) -> List[RankingUser]:
  if not type in ['quantity', 'page']:
    raise HTTPException(status_code=400, detail='invalid query')
  now = datetime.now()

  if type == 'quantity':
    q = db.query(models.User.user_id, models.User.name, func.sum(models.ReadPage.quantity).label('val'))
  if type == 'page':
    q = db.query(models.User.user_id, models.User.name, func.sum(models.ReadPage.pages).label('val'))

  q = q.filter(models.ReadPage.year == now.year, models.ReadPage.month == now.month, models.User.user_id == models.ReadPage.user_id).group_by(models.User.user_id, models.User.name).order_by(desc('val'))

  ranking = []
  for uid, name, val in q:
    ranking.append(RankingUser(
      user_id=uid,
      name=name,
      value=val
    ))

  return ranking

def get_total_info(db: Session, user_id: str) -> TotalInfo:
  quantity, page = db.query(func.count(models.Book.isbn), func.sum(models.Book.page)).filter(models.Book.isbn == models.Read.isbn, models.Read.user_id == user_id).first()
  return TotalInfo(
    user_id = user_id,
    quantity = quantity,
    page = page
  )