#!/usr/bin/env python

"""
API docs:
    https://github.com/npm/registry/blob/master/docs/REGISTRY-API.md
API base URL:
    http://registry.npmjs.com/

extra metrics:
    https://api-docs.npms.io/#api-Package-GetMultiPackageInfo

"""

from __future__ import print_function

import urllib

from .base import *
from stutils import decorators as d

class Package(BasePackage):
    base_url = 'http://registry.npmjs.com/'
    info = None  # stores cached package info

    @classmethod
    def all(cls, cache_file=None):
        if isinstance(cache_file, six.string_types):
            fh = open(cache_file, 'rb')
        elif isinstance(cache_file, six.StringIO):
            fh = cache_file
        else:
            # how to create cache file: wget -O npm.json <url below>
            # it is 14Gb as of Jan 2019
            fh = urllib.urlopen(
                'https://skimdb.npmjs.com/registry/_all_docs?include_docs=true')

        # requires yajl-tools: apt-get install yajl-tools
        import ijson.backends.yajl2 as ijson

        for package_info in ijson.items(fh, 'rows.item'):
            package_name = package_info['id']
            yield Package(package_name, info=package_info['doc'])

    def __init__(self, name, info=None):
        """
        Args:
            name (str): package name.
                Expect it will be normalized to prevent duplicate packages.
            info (Optional[dict]): in some cases (I'm talking about you, npm)
                ecosystem API lists packages with their metadata, so it's
                cheaper to reuse it.
                End users should not use this parameter.
        """
        if info:
            self.info = info
        else:
            try:
                self.info = self._request(name).json()
            except IOError:
                raise PackageDoesNotExist(
                    "Package %s does not exist or not public" % name)

        super(Package, self).__init__(name)

    @d.cached_property
    def _extra_info(self):
        return requests.get(
            "https://api.npms.io/v2/package/" +
            six.moves.urllib.parse.quote_plus(self.name)).json()

    @property
    def quality(self):
        return json_path(self._extra_info, 'score', 'detail', 'quality')

    @property
    def popularity(self):
        return json_path(self._extra_info, 'score', 'detail', 'popularity')

    @property
    def maintenance_score(self):
        return json_path(self._extra_info, 'score', 'detail', 'maintenance')

    @classmethod
    def _request(cls, *path):
        if path and not (path[0].startswith('https://') or path[0].startswith('http://')):
            path = (cls.base_url,) + path

        for _ in range(3):
            try:
                r = requests.get("/".join(path), timeout=TIMEOUT)
            except requests.exceptions.Timeout:
                continue
            r.raise_for_status()
            return r
        raise IOError("Failed to reach %s. "
                      "Check your Internet connection." % cls.base_url)

    def releases(self, include_unstable=False, include_backports=False):
        """ Return package release labels

        Args:
            include_unstable (bool): whether to include releases including
                symbols other than dots and numbers.
            include_backports (bool): whether to include releases smaller in
                version than last stable release

        Returns:
             List[Tuple[str, str]]: (label, date), sorted by date
        """
        raise NotImplementedError

    def download_url(self, ver):
        """Get URL to package file of the specified version
        This function takes into account supported file types and their
        relative preference (e.g. wheel files before source packages)

        Args:
            ver (str): version string

        Returns:
            Optional[str]: url string if found, None otherwise
        """
        assert ver in self.info['releases']
        raise NotImplementedError

    def download(self, ver=None):
        """Download and extract the specified package version

        Args:
             ver (str): Version of the package

        Returns:
            Optional[str]: path to the folder with extracted package,
                None if download failed
        """
        raise NotImplementedError

    @d.cached_property
    def repository(self):
        """ Search for software repository URL

        Returns:
            Optional[str]: path to the folder with extracted package,
                None if download failed
        """
        raise NotImplementedError

    def dependencies(self, ver=None):
        """ Get technical dependencies

        Args:
            ver (Optional[str]): version string, latest version by default

        Returns:
            Dict[str, str]: dictionary of the form `{package: version}`;
                `version` might contain platform-specific modifiers, e.g.
                '>=3.12.1'. If not specified, `version will be `None`.
        """
        raise NotImplementedError

    def loc_size(self, ver):
        """ Get package size in LOC """
        raise NotImplementedError


import os
import cv2
import numpy as np
import random
from pycocotools.coco import COCO

import torch
from torch.utils.data import Dataset

dataset_path = '/opt/ml/input/data'
category_names = ['Background', 'UNKNOWN', 'General trash', 'Paper', 'Paper pack', 'Metal',
                    'Glass', 'Plastic', 'Styrofoam', 'Plastic bag', 'Battery', 'Clothing']

def get_classname(classID, cats):
    for i in range(len(cats)):
        if cats[i]['id'] == classID:
            return cats[i]['name']
    return "None"

class RecycleTrashDataset(Dataset):
    """Some Information about RecycleTrashDataset"""
    def __init__(self, data_dir, mode='train', transform=None, augmix=None, augmix_prob=0):
        super().__init__()
        self.mode = mode
        self.transform = transform
        self.augmix = augmix
        self.augmix_prob = augmix_prob
        self.coco = COCO(os.path.join(dataset_path, data_dir))

    def __getitem__(self, index: int):
        # dataset이 index되어 list처럼 동작
        image_id = self.coco.getImgIds(imgIds=index)
        image_infos = self.coco.loadImgs(image_id)[0]

        # cv2 를 활용하여 image 불러오기
        images = cv2.imread(os.path.join(dataset_path, image_infos['file_name']))
        images = cv2.cvtColor(images, cv2.COLOR_BGR2RGB).astype(np.uint8)

        if (self.mode in ('train', 'val')):
            ann_ids = self.coco.getAnnIds(imgIds=image_infos['id'])
            anns = self.coco.loadAnns(ann_ids)

            # Load the categories in a variable
            cat_ids = self.coco.getCatIds()
            cats = self.coco.loadCats(cat_ids)

            # masks : size가 (height x width)인 2D
            # 각각의 pixel 값에는 "category id + 1" 할당
            # Background = 0
            masks = np.zeros((image_infos["height"], image_infos["width"]))
            # Unknown = 1, General trash = 2, ... , Cigarette = 11
            for i in range(len(anns)):
                className = get_classname(anns[i]['category_id'], cats)
                pixel_value = category_names.index(className)
                masks = np.maximum(self.coco.annToMask(anns[i]) * pixel_value, masks)
            masks = masks.astype(np.float32)

            # one-hot
            target = np.unique(masks).astype(int)
            one_hot_label = np.sum(np.eye(12)[target], axis=0)

            # img, mask 다 np , 512x512
            if self.augmix is not None :
              r = np.random.rand(1)
              if r < self.augmix_prob :
                  images , masks = self.augmix_search(images , masks)

            # transform -> albumentations 라이브러리 활용
            if self.transform is not None:
                transformed = self.transform(image=images, mask=masks)
                images = transformed["image"]
                masks = transformed["mask"]

            return images, masks, torch.Tensor(one_hot_label), image_infos

        if self.mode == 'test':
            # transform -> albumentations 라이브러리 활용
            if self.transform is not None:
                transformed = self.transform(image=images)
                images = transformed["image"]

            return images, image_infos

    def __len__(self) -> int:
        # 전체 dataset의 size를 return
        return len(self.coco.getImgIds())

    def augmix_search(self, images, masks):
        # image 3, 512, 512 ,mask: 512, 512 (둘 다 numpy)
        temp_dict = {
        0: 4, 1: 5, 2: 6, 3: 10, 4: 11
        }
        num = [ 4, 5, 6, 10, 11]

        label = random.choice(num)  # ex) 4
        idx = np.random.randint(len(self.augmix[label]))
        augmix_img = self.augmix[label][idx]

        augmix_mask = np.zeros((512, 512))
        augmix_mask[augmix_img[:, :, 0] != 0] = label     # augmix img가 있는 만큼 label로 mask를 채워줌

        images[augmix_img != 0] = augmix_img[augmix_img != 0]
        masks[augmix_mask!= 0] = augmix_mask[augmix_mask != 0]

        return images, masks

import pandas as pd
import albumentations as A

class PseudoDataset(Dataset):
    def __init__(self, data_dir='/opt/ml/input/data/', pseudo_csv='presudo_result.csv', transform=None):
        self.data_dir = data_dir
        self.df = pd.read_csv(os.path.join(data_dir, pseudo_csv))
        self.tfms = A.Resize(512, 512, interpolation= cv2.INTER_NEAREST)

        self.transform = transform

    def __len__(self):
        return len(self.df)

    def __getitem__(self, idx):
        img_path = self.df.iloc[idx, 0]
        img_path = os.path.join(self.data_dir, img_path)
        mask = self.df.iloc[idx, 1]
        mask = np.array(list(map(int, mask.split())))
        mask = mask.reshape(256, 256)
        mask = self.tfms(image=mask)['image']

        img = cv2.imread(img_path)
        img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB).astype(np.uint8)

        # one-hot
        target = np.unique(mask).astype(int)
        one_hot_label = np.sum(np.eye(12)[target], axis=0)

        if self.transform is not None:
            transformed = self.transform(image=img, mask=mask)
            images = transformed["image"]
            masks = transformed["mask"]

        return images, masks, torch.Tensor(one_hot_label), img_path

class AllDataset(Dataset):
    def __init__(self, data_dir, mode='train', augmix_prob=0.15, augmix=None, transform=None):
        self.presudo_transform = A.Compose([
            A.Resize(512, 512),
            A.Normalize(mean=(0.461, 0.440, 0.419), std=(0.211, 0.208, 0.216)),
            A.pytorch.transforms.ToTensorV2()
        ])
        self.presudo_ds = PseudoDataset(transform=self.presudo_transform)
        self.seg_ds = RecycleTrashDataset(data_dir=data_dir, mode=mode, augmix=augmix, augmix_prob=augmix_prob, transform=transform)

    def __len__(self):
        return len(self.presudo_ds) + len(self.seg_ds)

    def __getitem__(self, idx):
        if idx >= len(self.seg_ds):
            return self.presudo_ds.__getitem__(idx - len(self.seg_ds))
        else:
            return self.seg_ds.__getitem__(idx)


"""Whatsapp chat data loader"""

import logging
from pathlib import Path
from typing import List

from llama_index.readers.base import BaseReader
from llama_index.readers.schema.base import Document

class WhatsappChatLoader(BaseReader):
    """
    Whatsapp chat data loader.

    Args:
        path (str): Path to Whatsapp chat file.
    """

    def __init__(self, path: str):
        """Initialize with path."""

        self.file_path = path

    def load_data(self) -> List[Document]:
        """
        Parse Whatsapp file into Documents
        """

        from chatminer.chatparsers import WhatsAppParser

        path = Path(self.file_path)

        parser = WhatsAppParser(path)
        parser.parse_file()
        df = parser.parsed_messages.get_df()

        logging.debug(f"> Number of messages: {len(df)}.")

        docs = []
        n = 0
        for row in df.itertuples():
            extra_info = {
                "source": str(path).split("/")[-1].replace(".txt", ""),
                "author": row.author,
                "timestamp": str(row.timestamp),
            }

            docs.append(
                Document(
                    text=str(row.timestamp)
                    + " "
                    + row.author
                    + ":"
                    + " "
                    + row.message,
                    extra_info=extra_info,
                )
            )

            n += 1
            logging.debug(f"Added {n} of {len(df)} messages.")

        logging.debug(f"> Document creation for {path} is complete.")
        return docs


from django.urls import path
from rest_framework import routers
from rest_framework_simplejwt.views import TokenRefreshView

from .api import AuthViewSet, ListUsersViewSet, GetUserView
from .views import *

urlpatterns = [
    path("login/", login, name="login"),
    path("logout/", logout, name="logout"),
    path("register/", register, name="register"),
    path("settings/", settings, name="settings"),
    path("settings/tg_unlick", tg_unlink, name="tg_unlick"),
    path("", user_list, name="users"),
    path("profile/<str:username>/", profile, name="profile"),
    path("api/refresh", TokenRefreshView.as_view(), name="token_refresh"),
    path("api/list", ListUsersViewSet.as_view(), name="user_list"),
    path("api/list/<int:id>", GetUserView.as_view({"get": "list"}), name="user_api_id"),
]

router = routers.DefaultRouter(trailing_slash=False)
router.register("api", AuthViewSet, basename="api_auth")

urlpatterns += router.urls


import os
import re
import glob
import pandas as pd
from io import StringIO

from django.conf import settings
from loguru import logger

description_split_text = 'DATA TRANSAKCJI:'

def extract_date_from_description(x):
    parts = re.split(description_split_text, x)
    if len(parts) > 1:
        return parts[1].strip()

    return None

def trim_description(x):
    parts = re.split(description_split_text, x)
    if len(parts) > 1:
        return parts[0].strip()

    return x

def trim_text_fields(x):
    return ' '.join(str(x).split())

def load_csv():
    # load all csv files from temp dir
    source_dir = os.path.join(settings.BASE_DIR, 'temp')
    field_names = ['Date', 'Added', 'Type', 'Description', 'Party Name', 'Party IBAN', 'Amount', 'Balance',
                   'Account Number']
    df = pd.DataFrame()

    list_of_files = glob.glob(os.path.join(source_dir, '*.csv'))
    for this_file in list_of_files:
        logger.info("adding {}".format(this_file))
        this_account_number = ''
        this_data = ''
        save_account_number = False
        save_transactions = False
        with open(this_file, encoding='cp1250') as f:
            for line in f:
                if line[:15] == '#Numer rachunku':
                    save_account_number = True
                elif line[:14][:9] == '#Data ksi':
                    save_transactions = True
                elif save_account_number:
                    this_account_number = line[:32].replace(' ', '')
                    save_account_number = False
                elif save_transactions:
                    if line[0] == '2':
                        this_data += line.replace("'", '"').replace('";"', "|;|").\
                            replace('";', "|;").replace(';"', ";|")
                    else:
                        save_transactions = False

        df_temp = pd.read_csv(StringIO(this_data), sep=';', comment='#', engine='python', names=field_names,
                              quotechar='|', encoding='cp1250', dtype={'Party IBAN': 'str'})

        logger.warning(this_file);
        logger.warning(this_data);
        logger.warning(this_account_number);
        # logger.info(df_temp);

        df_temp['Account Number'] = this_account_number
        df_temp['Source'] = os.path.basename(this_file)
        df_temp['Description'] = df_temp['Description'].fillna('')
        df_temp['Party IBAN'] = df_temp['Party IBAN'].fillna('')
        df_temp['Party Name'] = df_temp['Party Name'].fillna('')
        df_temp['Type'] = df_temp['Type'].fillna('')

        df_temp['Date Modified'] = df_temp['Description'].map(extract_date_from_description)
        df_temp['Description'] = df_temp['Description'].map(trim_text_fields)
        df_temp['Party Name'] = df_temp['Party Name'].map(trim_text_fields)
        df_temp['Type'] = df_temp['Type'].map(trim_text_fields)

        df_temp['Date Modified'] = df_temp['Date Modified'].fillna(df_temp['Date'])
        df_temp['Amount Modified'] = df_temp['Amount'].map(
            lambda x: float(re.sub('[^0-9\,\.-]','', x).replace(',', '.')))
        df_temp['Balance Modified'] = df_temp['Balance'].map(
            lambda x: float(re.sub('[^0-9\,\.-]', '', x).replace(',', '.')))

        df = df.append(df_temp, ignore_index=True)
        logger.info("added {}".format(df_temp.shape))

    logger.info("whole DF shape: {}".format(df.shape))
    logger.info(df.info())

    return df


"""
Cordy early / late LI analysis.
Compare target preference at the end of the day vs. the beginnig of the day for
days where only one set of targets was used (started on 11/22/2019)
"""
import nems.db as nd
import datetime as dt
import numpy as np
import matplotlib.pyplot as plt
import pandas as pd
from nems_lbhb.baphy_experiment import BAPHYExperiment

runclass = 'BVT'
animal = 'Cordyceps'
an_regex = "%"+animal+"%"
min_trials = 50
earliest_date = '2019_11_22'
window_length = 50 # how many trials in beginning / end to include

# get list of all training parmfiles
sql = "SELECT parmfile, resppath FROM gDataRaw WHERE runclass=%s and resppath like %s and training = 1 and bad=0 and trials>%s and behavior='active'"
parmfiles = nd.pd_query(sql, (runclass, an_regex, min_trials))

# screen for dates
parmfiles['date'] = [dt.datetime.strptime('-'.join(x.split('_')[1:-2]), '%Y-%m-%d') for x in parmfiles.parmfile]
ed = dt.datetime.strptime(earliest_date, '%Y_%m_%d')
parmfiles = parmfiles[parmfiles.date > ed]

results = pd.DataFrame(index=parmfiles['date'].unique(), columns=['Early', 'Late', 'Overall', 'nTrials'])
options = {}
for date in parmfiles['date'].unique():
    files = [p for i, p in parmfiles.iterrows() if p.date==date]
    files = [f['resppath']+f['parmfile'] for f in files]

    manager = BAPHYExperiment(files)
    events = manager.get_behavior_events(correction_method='baphy', **options)
    events = manager._stack_events(events)

    good_trials = events[(~events.invalidTrial)].Trial.unique()
    # note, number of target trials in performance dict doesn't have to add up to
    # number of window_length trials. There could (definitely are) FA trials
    # that are "valid trials", but not counted for behavior analysis bc target never
    # played

    params = manager.get_baphy_exptparams()[0]
    targets = np.array(params['TrialObject'][1]['TargetHandle'][1]['Names'])
    pump_dur = np.array(params['BehaveObject'][1]['PumpDuration'])
    r_tar = targets[pump_dur>0][0]
    nr_tar = targets[pump_dur==0][0]

    if len(good_trials) < (window_length * 1.5):
        all_trials = manager.get_behavior_performance(trials=None, **options)
        results.loc[date]['Overall'] = (all_trials['RR'][r_tar] - all_trials['RR'][nr_tar]) / \
                                    (all_trials['RR'][r_tar] + all_trials['RR'][nr_tar])
    else:
        all_trials = manager.get_behavior_performance(trials=None, **options)
        results.loc[date]['Overall'] = (all_trials['RR'][r_tar] - all_trials['RR'][nr_tar]) / \
                                            (all_trials['RR'][r_tar] + all_trials['RR'][nr_tar])
        results.loc[date]['nTrials'] = len(good_trials)

        # get LI for first 50 trials
        early_trials = good_trials[:window_length]
        early = manager.get_behavior_performance(trials=early_trials, **options)

        # get LI for last 50 trials
        late_trials = good_trials[-window_length:-10]
        late = manager.get_behavior_performance(trials=late_trials, **options)

        results.loc[date]['Early'] = (early['RR'][r_tar] - early['RR'][nr_tar]) / (early['RR'][r_tar] + early['RR'][nr_tar])
        results.loc[date]['Late'] = (late['RR'][r_tar] - late['RR'][nr_tar]) / (late['RR'][r_tar] + late['RR'][nr_tar])

# compare early / late learning index and show overall distribution of LI
f, ax = plt.subplots(1, 2, figsize=(8, 4))

ax[0].bar([0, 1], [results['Early'].mean(), results['Late'].mean()],
                yerr=[results['Early'].sem(), results['Late'].sem()],
                color='lightgrey', edgecolor='k')
ax[0].set_ylabel('Learning Index')
ax[0].set_xticks([0, 1])
ax[0].set_xticklabels(['Early', 'Late'])

bins = np.arange(-.5, .5, 0.05)
results['Overall'].plot.hist(edgecolor='white', bins=bins, ax=ax[1])
ax[1].set_xlabel('Learning Index')
ax[1].set_title('All Valid Trials')

f.tight_layout()

plt.show()


import sqlite3
from model.student_model import Student

# A class that provides configuration to setup and manipulate DB of the final project
class DBUtil:
    __DB_NAME = 'finalProject.db'
    __DROP_IF_EXISTS_STUDENTS = '''DROP Table IF EXISTS Students'''
    __CREATE_TABLE_STUDENTS = '''CREATE TABLE Students(
                                    registration INTEGER NOT NULL PRIMARY KEY,
                                    name TEXT NOT NULL,
                                    email TEXT NOT NULL,
                                    dBirth TEXT NOT NULL,
                                    hometown TEXT NOT NULL,
                                    score REAL NOT NULL)
                                '''
    __INSERT_STUDENT_QUERY = '''INSERT OR IGNORE INTO Students
                                    (registration, name, email, dBirth, hometown, score)
                                    VALUES(?, ?, ?, date(?), ?, ?)
                                '''
    __DELETE_STUDENT_QUERY = '''DELETE FROM Students
                                WHERE registration = ?
                                '''
    __GET_STUDENT_BY_ID_QUERY = '''SELECT * FROM Students
                                WHERE registration = ? ORDER BY registration
                            '''
    __GET_ALL_STUDENTS_QUERY = '''SELECT * FROM Students ORDER BY registration
                                '''
    __GET_ALL_STUDENTS_PAGING_QUERY = '''SELECT * FROM Students ORDER BY registration
                                            LIMIT ? OFFSET ?
                                        '''
    __GET_STUDENTS_PAGING_QUERY = '''SELECT DISTINCT * FROM Students
                                            WHERE registration = ? OR
                                                  name like ? OR
                                                  email like ? OR
                                                  dBirth = ? OR
                                                  hometown like ? OR
                                                  score = ?
                                            ORDER BY score, name DESC
                                            LIMIT ? OFFSET ?
                                            '''
    __UPDATE_STUDENT_QUERY = '''UPDATE Students
                                    SET registration = ?, name = ?, email = ?, dBirth = ?, hometown = ?, score = ?
                                    WHERE registration = ?
                                '''

    def __get_connection(self):
        return sqlite3.connect(self.__DB_NAME)

    def create_table_students(self):
        db = self.__get_connection()
        cur = db.cursor()
        cur.execute(self.__DROP_IF_EXISTS_STUDENTS)
        cur.execute(self.__CREATE_TABLE_STUDENTS)
        cur.close()
        db.commit()
        db.close()

    def insert_student(self, student: Student):
        db = self.__get_connection()
        cur = db.cursor()
        cur.execute(self.__INSERT_STUDENT_QUERY,
                    (student.registration,
                     student.name,
                     student.email,
                     student.dBirth,
                     student.hometown,
                     student.score,))
        cur.close()
        db.commit()
        db.close()

    def update_student(self, id,  student: Student):
        db = self.__get_connection()
        cur = db.cursor()
        cur.execute(self.__UPDATE_STUDENT_QUERY,
                    (student.registration,
                     student.name,
                     student.email,
                     student.dBirth,
                     student.hometown,
                     student.score,
                     id))
        cur.close()
        db.commit()
        db.close()

    def delete_student_by_registration(self, registration: int):
        db = self.__get_connection()
        cur = db.cursor()
        cur.execute(self.__DELETE_STUDENT_QUERY,
                    (registration,))
        self.__db_commit_and_close(db)

    def get_students_by_registration(self, registration: int):
        db = self.__get_connection()
        cur = db.cursor()
        cur.execute(self.__GET_STUDENT_BY_ID_QUERY,
                    (registration,))
        return cur.fetchone()

    def get_all_students(self):
        db = self.__get_connection()
        cur = db.cursor()
        cur.execute(self.__GET_ALL_STUDENTS_QUERY)
        return cur.fetchall()

    def get_all_students_by_pagination(self, limit=1, offset=0):
        db = self.__get_connection()
        cur = db.cursor()
        cur.execute(self.__GET_ALL_STUDENTS_PAGING_QUERY, (limit, offset))
        return cur.fetchall()

    def get_students_by_pagination(self, criteria, num, limit=1, offset=0):
        db = self.__get_connection()
        cur = db.cursor()
        cur.execute(self.__GET_STUDENTS_PAGING_QUERY, (num,
                                                       criteria,
                                                       criteria,
                                                       criteria,
                                                       criteria,
                                                       num,
                                                       limit,
                                                       offset))
        return cur.fetchall()

    def __db_commit_and_close(self, db):
        cur = db.cursor()
        db.commit()
        db.close()


from datetime import datetime, timedelta
import vim

start_time = None
end_time = None

VIMODORO_INTERVAL = 1 # minutes

def vimodoro():
    if _is_timer_running():
        _start_vimodoro()
    else:
        _stop_vimodoro()

def _is_timer_running():
    return start_time is None and end_time is None

def _start_vimodoro():
    global start_time, end_time
    start_time = datetime.now()
    end_time = start_time + timedelta(minutes=VIMODORO_INTERVAL)

def _stop_vimodoro():
    # Reset the times
    global start_time, end_time
    start_time = end_time = None

def get_time_remaining():
    if end_time is None:
        vim.command(f"let full_time = '00:00'")
        return

    current_time = datetime.now()

    if current_time > end_time:
        seconds = 0
    else:
        time_left = end_time - current_time
        seconds = time_left.seconds

    minutes_left = seconds // 60
    seconds_left = seconds % 60
    full_time = str(minutes_left) + ':' + str(seconds_left)
    vim.command(f"let full_time = '{full_time}'")


import configparser
import random
import time

from common.util.html import get_html
from source.data import Data

def lj_spider(config, data):
    # 获取地址
    lj_area = config.get('website', 'lj_area')
    url = config.get('website', 'lj')

    for area in lj_area.split(','):
        # 区域Url格式化
        req_url = config.get('website', 'req_lj').format(area)
        # 爬取数据
        html = get_html(req_url)
        # 保存数据，获取最大页码
        max_page = data.lj_save(html, url, req_url, True)

        # 大于1页，遍历爬取
        for i in range(max_page + 1):
            if i > 1:
                req_url_pg = config.get('website', 'req_lj_pg').format(area, str(i))
                html = get_html(req_url_pg)
                data.lj_save(html, url, req_url, False)

        # 暂停时间，模拟人工操作
        time.sleep(random.randint(1, 10))

def bk_spider(config, data):
    bk_area = config.get('website', 'bk_area')
    url = config.get('website', 'bk')

    for area in bk_area.split(','):
        req_url = config.get('website', 'req_bk').format(area)
        html = get_html(req_url)
        max_page = data.bk_save(html, url, req_url, True)

        for i in range(max_page + 1):
            if i > 1:
                req_url_pg = config.get('website', 'req_bk_pg').format(area, str(i))
                html = get_html(req_url_pg)
                data.bk_save(html, url, req_url, False)

        time.sleep(random.randint(1, 10))

if __name__ == '__main__':
    print('Spider-Man!!')

    # 获取配置
    config = configparser.ConfigParser()
    config.read("config.ini")

    # 获取数据源
    data = Data(config)

    # lj_spider(config, data)
    bk_spider(config, data)


import logging
import polib
from openai import OpenAI

import logging
from translator.translate_service import TranslateService

from translator.logger import format_log

logging.basicConfig(level=logging.INFO, format="%(message)s")
logging.getLogger("httpx").setLevel(logging.WARNING)

if __name__ == "__main__":
    input_file = "/home/oskar/git/translator/futurecoder_English.po"
    output_file = "/home/oskar/git/translator/futurecoder_Polish.po"

    po = polib.pofile(input_file)
    client = OpenAI()
    total_entries = len(po)
    translate_service = TranslateService(client, total_entries)
    for index, entry in enumerate(po):
        logging.info(format_log(entry.msgstr, "INPUT"))
        translated = translate_service.translate(entry.msgstr, index)
        entry.msgstr = translated
        logging.info(format_log(entry.msgstr, "OUTPUT"))
        po.save(output_file)


from setuptools import find_packages
from setuptools import setup

REQUIRED_PACKAGES = [
    "astropy",
    "librosa >= 0.9.2",
    "numpy == 1.24.4",
    "pandas >= 1.5.2",
    "pretty_midi >= 0.2.9",
    "scipy >= 1.11.1",
    "tensorflow >= 2.12.0",
    "urllib3 < 2.0",
]

setup(
    name="jjf",
    version="0.0.1",  # pylint: disable=undefined-variable
    description="Use machine learning to create art and music",
    long_description="",
    author="Aaron Brenzel",
    author_email="aaronbrenzel@gmail.com",
    license="Apache 2",
    keywords="note_seq note sequences",
    packages=find_packages(),
    package_data={"jjf": ["*.pyi", "**/*.pyi"]},
    install_requires=REQUIRED_PACKAGES,
    setup_requires=["pytest-runner", "pytest-pylint"],
    tests_require=[
        "pytest >= 5.2.0",
        "pytest-xdist < 1.30.0",  # 1.30 has problems working with pylint plugin
        "pylint >= 2.4.2",
    ],
)


# -*- coding: utf-8 -*-
"""Propagating cumulative surface hopping trajectories"""

from __future__ import division

import numpy as np

from .trajectory_sh import TrajectorySH

from typing import Any, List, Dict
from .typing import ArrayLike

class TrajectoryCum(TrajectorySH):
    """
    Trajectory surface hopping using a cumulative approach rather than instantaneous

    Instead of using a random number generator at every time step to test for a hop,
    hops occur when the cumulative probability of a hop crosses a randomly determined
    threshold. Swarmed results should be identical to the traditional variety, but
    should be a bit easier to reproduce since far fewer random numbers are ever needed.
    """

    def __init__(self, *args: Any, **kwargs: Any):
        """Constructor (see TrajectorySH constructor)"""
        TrajectorySH.__init__(self, *args, **kwargs)

        self.prob_cum = np.longdouble(0.0)
        self.zeta = self.draw_new_zeta()

    def snapshot(self) -> Dict:
        """returns loggable data"""
        out = {
            "time": self.time,
            "position": self.position.tolist(),
            "momentum": (self.mass * self.velocity).tolist(),
            "potential": self.potential_energy().item(),
            "kinetic": self.kinetic_energy().item(),
            "energy": self.total_energy().item(),
            "density_matrix": self.rho.view(np.float64).tolist(),
            "active": int(self.state),
            "electronics": self.electronics.as_dict(),
            "hopping": float(self.hopping),
            "zeta": float(self.zeta),
            "prob_cum": float(self.prob_cum)
        }
        return out

    def hopper(self, gkndt: ArrayLike) -> List[Dict]:
        """given a set of probabilities, determines whether and where to hop

        :param probs: [nstates] numpy array of individual hopping probabilities
        :returns: dictioanry with { "target": int, "weight": float, "zeta": float, "prob": float }
        """
        self.hopping = np.sum(gkndt)

        accumulated = np.longdouble(self.prob_cum)
        gkdt = np.sum(gkndt)
        accumulated += (accumulated - 1.0) * np.expm1(-gkdt)
        if accumulated > self.zeta:  # then hop
            # where to hop
            hop_choice = gkndt / gkdt

            zeta = self.zeta
            target = self.random_state.choice(list(range(self.model.nstates())), p=hop_choice)

            # reset probabilities and random
            self.prob_cum = 0.0
            self.zeta = self.draw_new_zeta()

            return [{"target": target, "weight": 1.0, "zeta": zeta, "prob": accumulated}]

        self.prob_cum = accumulated
        return []


from pptx import Presentation
import textwrap

def shorten_text(text, max_length):
    """
    Shorten the text to the max_length, adding an ellipsis if the text is shortened.
    """
    return textwrap.shorten(text, width=max_length, placeholder="...")

def process_slide(slide, max_length):
    """
    Process each slide, shortening the text in each text box if it's too long.
    """
    for shape in slide.shapes:
        if not shape.has_text_frame:
            continue
        text_frame = shape.text_frame
        for paragraph in text_frame.paragraphs:
            if len(paragraph.text) > max_length:
                paragraph.text = shorten_text(paragraph.text, max_length)

def process_presentation(input_file, output_file, max_length):
    """
    Process each slide in the presentation.
    """
    ppt = Presentation(input_file)
    for slide in ppt.slides:
        process_slide(slide, max_length)
    ppt.save(output_file)

# Example usage
input_file = 'path/to/input/presentation.pptx'
output_file = 'path/to/output/presentation.pptx'
max_length = 50  # Maximum length of text in characters
process_presentation(input_file, output_file, max_length)


from fastapi import FastAPI
from fastapi.encoders import jsonable_encoder
from fastapi.exceptions import RequestValidationError
from fastapi.responses import JSONResponse
from fastapi.testclient import TestClient
from pytest import mark

from controllers.cart_items_routes import router as cart_items_router
from controllers.cart_routes import router as cart_router
from controllers.product_routes import router as product_router
from controllers.user_routes import Request
from controllers.user_routes import router as user_router
from controllers.user_routes import status
from project_logs.logging import set_logging
from server.database import get_db
from server.database_test import drop_databases_to_test, get_db as get_test_db
from utils.generate_fakes import (
    generate_fake_cart,
    generate_fake_cart_item,
    generate_fake_products,
    generate_fake_user,
)

app = FastAPI()
app.include_router(user_router, tags=["user"], prefix="/user")
app.include_router(product_router, tags=["products"], prefix="/products")
app.include_router(cart_router, tags=["cart"], prefix="/cart")
app.include_router(cart_items_router, tags=["cart_item"], prefix="/cart/{cart_id}/item")

app.dependency_overrides[get_db] = get_test_db

@app.on_event("shutdown")
async def shutdown_db_client():
    await drop_databases_to_test()

@app.exception_handler(RequestValidationError)
async def validation_exception_handler(request: Request, exc: RequestValidationError):
    project_errors = []
    project_error = {}
    error_log = set_logging("errors")
    for error in exc.errors():
        project_error["error_loc"] = error["loc"]
        project_error["error_type"] = error["type"]
        project_error["error_msg"] = error["msg"]
        project_errors.append(project_error)
        error_log.error(project_error)
    return JSONResponse(
        status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
        content=jsonable_encoder(project_errors),
    )

@mark.asyncio
async def test_create_update_cart_item():
    with TestClient(app) as client:
        body_client = await generate_fake_user(client)
        fake_products = await generate_fake_products(client)
        user_id = body_client.json().get("_id")
        fake_cart = await generate_fake_cart(client, user_id)
        quantity = 1
        response = await generate_fake_cart_item(
            client, fake_cart.json(), fake_products.json()[0], quantity
        )
        get_cart_response = client.get("/cart/" + user_id)
        assert response.status_code == 201
        body = response.json()
        assert body.get("cart").get("_id") == fake_cart.json().get("_id")
        assert body.get("cart").get("price") == 0
        assert body.get("product").get("price") == 9.99
        assert get_cart_response.json().get("price") == 9.99 * quantity

@mark.asyncio
async def test_create_update_cart_item_with_more_quantity():
    with TestClient(app) as client:
        body_client = await generate_fake_user(client)
        fake_products = await generate_fake_products(client)
        user_id = body_client.json().get("_id")
        fake_cart = await generate_fake_cart(client, user_id)
        quantity = 20
        response = await generate_fake_cart_item(
            client, fake_cart.json(), fake_products.json()[0], quantity
        )
        get_cart_response = client.get("/cart/" + user_id)
        assert response.status_code == 201
        body = response.json()
        assert body.get("cart").get("_id") == fake_cart.json().get("_id")
        assert body.get("cart").get("price") == 0
        assert body.get("product").get("price") == 9.99
        assert get_cart_response.json().get("price") == 9.99 * quantity

@mark.asyncio
async def test_create_update_cart_item_missing_fields():
    with TestClient(app) as client:
        body_client = await generate_fake_user(client)
        fake_products = await generate_fake_products(client)
        user_id = body_client.json().get("_id")
        fake_cart = await generate_fake_cart(client, user_id)
        quantity = 1
        await generate_fake_cart_item(
            client, fake_cart.json(), fake_products.json()[0], quantity
        )
        response = client.put(
            f"/cart/{body_client.json().get('_id')}/item/",
            json={
                "cart": fake_cart.json().get("_id"),
                "product": {
                    "description": "Doce gelado de morango",
                    "price": 9.99,
                },
            },
        )
        assert response.status_code == 422

@mark.asyncio
async def test_get_cart_item():
    with TestClient(app) as client:
        body_client = await generate_fake_user(client)
        fake_products = await generate_fake_products(client)
        user_id = body_client.json().get("_id")
        first_product_id = fake_products.json()[0].get("_id")
        fake_cart = await generate_fake_cart(client, user_id)
        await generate_fake_cart_item(client, fake_cart.json(), fake_products.json()[0])
        get_cart_item_response = client.get(
            "/cart/" + fake_cart.json().get("_id") + "/item/" + first_product_id + "/"
        )
        assert get_cart_item_response.status_code == 200
        assert get_cart_item_response.json().get("cart") == fake_cart.json()
        assert get_cart_item_response.json().get("product") == fake_products.json()[0]

@mark.asyncio
async def test_get_cart_unexisting():
    with TestClient(app) as client:
        get_cart_response = client.get("/cart/unexisting_id")
        assert get_cart_response.status_code == 404

@mark.asyncio
async def test_delete_cart_item():
    with TestClient(app) as client:
        body_client = await generate_fake_user(client)
        fake_products = await generate_fake_products(client)
        user_id = body_client.json().get("_id")
        first_product_id = fake_products.json()[0].get("_id")
        fake_cart = await generate_fake_cart(client, user_id)
        await generate_fake_cart_item(client, fake_cart.json(), fake_products.json()[0])
        delete_cart_item_response = client.delete(
            f"""/cart/{fake_cart.json().get("_id")}/item/{first_product_id}"""
        )
        assert delete_cart_item_response.status_code == 303

@mark.asyncio
async def test_delete_cart_item_with_more_quantity():
    with TestClient(app) as client:
        body_client = await generate_fake_user(client)
        fake_products = await generate_fake_products(client)
        user_id = body_client.json().get("_id")
        first_product_id = fake_products.json()[0].get("_id")
        fake_cart = await generate_fake_cart(client, user_id)
        await generate_fake_cart_item(
            client, fake_cart.json(), fake_products.json()[0], 4
        )
        delete_cart_item_response = client.delete(
            f"""/cart/{fake_cart.json().get("_id")}/item/{first_product_id}"""
        )
        assert delete_cart_item_response.status_code == 200

@mark.asyncio
async def test_delete_cart_item_unexisting():
    with TestClient(app) as client:
        delete_cart_response = client.delete("/cart/unexisting_id/item/10")
        assert delete_cart_response.status_code == 404

@mark.asyncio
async def test_delete_user_cascade():
    with TestClient(app) as client:
        body_client = await generate_fake_user(client)
        fake_products = await generate_fake_products(client)
        user_id = body_client.json().get("_id")
        fake_cart = await generate_fake_cart(client, user_id)
        await generate_fake_cart_item(
            client, fake_cart.json(), fake_products.json()[0], 4
        )
        delete_user_response = client.delete("/user/" + user_id)
        assert delete_user_response.status_code == 303
        get_user_response = client.get("/user/" + user_id)
        assert get_user_response.status_code == 404
        get_cart_response = client.get("/cart/" + user_id)
        assert get_cart_response.status_code == 404
        get_cart_item_response = client.get("/cart/" + fake_cart.json().get("_id"))
        assert get_cart_item_response.status_code == 404

@mark.asyncio
async def test_delete_cart_cascade():
    with TestClient(app) as client:
        body_client = await generate_fake_user(client)
        fake_products = await generate_fake_products(client)
        user_id = body_client.json().get("_id")
        fake_cart = await generate_fake_cart(client, user_id)
        await generate_fake_cart_item(
            client, fake_cart.json(), fake_products.json()[0], 4
        )
        delete_cart_response = client.delete("/cart/" + user_id)
        assert delete_cart_response.status_code == 200
        assert delete_cart_response.json().get("status") == "success"
        assert delete_cart_response.json().get("msg") == "Deleted the user cart"
        get_cart_response = client.get("/cart/" + user_id)
        assert get_cart_response.status_code == 404
        get_cart_item_response = client.get("/cart/" + fake_cart.json().get("_id"))
        assert get_cart_item_response.status_code == 404

if __name__ == "__main__":
    pass


import base64

def pic2py(picture_name):
    open_pic = open("%s" % picture_name, 'rb')
    b64str = base64.b64encode(open_pic.read())
    open_pic.close()

    # 注意这边b64str一定要加上.decode()
    write_data = 'img = "%s"' % b64str.decode()
    f = open('%s.py' % picture_name.replace('.', '_'), 'w+')
    f.write(write_data)
    f.close()

if __name__ == '__main__':
    name = input(r"C:\Users\nimendafeige\PycharmProjects\untitled\1.gif")
    pic2py(name)


from typing import List

def checkPossibility(nums: List[int]) -> bool:
	# if len(nums) < 1:
	# 	return True
	# count = 0
	# for i in range(1, len(nums)):
	# 	if count >= 2:
	# 		return False
	# 	if nums[i - 1] < nums[i]:
	# 		continue;
	# 	else:
	# 		count += 1
	# if count < 2:
	# 	return True
	# else:
	# 	return False
	check = False
	for i in range(0, len(nums) - 1):
		if (nums[i] > nums[i + 1]) and check == True:
			return False
		elif nums[i] > nums[i + 1]:
			if i!=0 and nums[i - 1] > nums[i + 1]:
				nums[i + 1] = nums[i]
			check = True
	return True

li = [4,2,3]
li2 = [3,4,2,3]
print(checkPossibility(li))
print(checkPossibility(li2))


from dependency_injector.wiring import Provide, inject
from fastapi import APIRouter, Depends
from pydantic import BaseModel

from app.container import AppContainer
from app.core.jwt import get_current_user

from .usecases import (
    CreateThreadUsecase,
    GetThreadMembers,
    GetThreadUsecase,
    GetUserThreads,
    RemoveMemberFromThreadUsecase,
)

Container = AppContainer.thread
router = APIRouter(prefix="/threads", tags=["threads"])

@router.get("")
@inject
def get_all_threads(
    user_id: int = Depends(get_current_user),
    usecase: GetUserThreads = Depends(Provide[Container.get_user_threads]),
):
    threads = usecase.get_user_threads(user_id)
    return {"data": threads}

class CreateThreadRequest(BaseModel):
    name: str

@router.post("")
@inject
def create_thread(
    req: CreateThreadRequest,
    user_id: int = Depends(get_current_user),
    usecase: CreateThreadUsecase = Depends(Provide[Container.create_thread]),
):
    thread = usecase.create_thread(req.name, user_id)
    return {"data": thread}

@router.get("/{thread_id}")
@inject
def get_thread(
    thread_id: int,
    user_id: int = Depends(get_current_user),
    usecase: GetThreadUsecase = Depends(Provide[Container.get_thread]),
):
    thread = usecase.get_thread_for_user(thread_id, user_id)
    return {"data": thread}

@router.get("/{thread_id}/members")
@inject
def get_thread_members(
    thread_id: int,
    user_id: int = Depends(get_current_user),
    usecase: GetThreadMembers = Depends(Provide[Container.get_thread_members]),
):
    thread_members = usecase.get_thread_members_for_user(thread_id, user_id)
    return {"data": thread_members}

class RemoveMemberRequest(BaseModel):
    user_id: int

@router.delete("/{thread_id}/members")
@inject
def remove_member_from_thread(
    thread_id: int,
    req: RemoveMemberRequest,
    user_id: int = Depends(get_current_user),
    usecase: RemoveMemberFromThreadUsecase = Depends(
        Provide[Container.remove_member_from_thread]
    ),
):
    usecase.remove_member(thread_id, user_id, req.user_id)
    return {"data": {"success": True}}


from flask_bootstrap import Bootstrap
from flask_sqlalchemy import SQLAlchemy
from flask_login import LoginManager
from flask_cache import Cache
from flask_migrate import Migrate
from  flask_moment import Moment
from flask_uploads import UploadSet,configure_uploads,patch_request_class,IMAGES

bootstrap = Bootstrap()
moment = Moment()
db = SQLAlchemy()
migrate = Migrate(db=db)
login_manager = LoginManager()
cache = Cache(config={'CACHE_TYPE':'redis'})

file = UploadSet('photos',IMAGES)

def ext_init(app):
    bootstrap.init_app(app)
    moment.init_app(app)
    db.init_app(app)
    migrate.init_app(app)
    cache.init_app(app)
    configure_uploads(app, file)
    patch_request_class(app, None)

    login_manager.init_app(app)
    login_manager.login_view = 'user.login'
    login_manager.login_message = '先登录再访问'
    login_manager.session_protection = 'strong'


import json
import os
import openai
from flask import Flask, request, jsonify, render_template

app = Flask(__name__)
promptdefault = "detailed image"

@app.route('/')
def home():
    return "Hello, Flask!"

@app.route('/prompt', methods =["POST", "GET"])
def prompting():
    if request.method == "POST":
       story_prompt = request.form.get("sprompt")
       # getting input with name = name in HTML form
       character_name = request.form.get("cname")
       global promptdefault
       promptdefault = story_prompt
       img_url = generate_image()
       return render_template("testing.html", image_url=img_url)
    return render_template("prompting.html")

@app.route('/testing', methods=['POST', 'GET'])
def loading():
    return render_template('testing.html')

@app.route('/generate-image', methods=['POST', 'GET'])
def generate_image():
    openai.api_key = os.getenv("DALLE_API_KEY")
    global promptdefault

    if not openai.api_key:
        return jsonify({'error': 'API key not found'}), 403

    try:
        # Call the OpenAI API to generate an image
        response = openai.Image.create(
            prompt=promptdefault,
            n=1,
            size="1024x1024"
        )

        # Extract the image URL(s) from the response
        # Adjust the following line according to the actual response structure
        image_urls = [data['url'] for data in response['data']]
        # Send back the image URLs in the response
        #jsonify({'images': image_urls})
        return image_urls[0]
        #return render_template("testing.html", image_url=website_url)

    except openai.error.OpenAIError as e:
        # Handle OpenAI specific errors
        return jsonify({'error': str(e)}), 500
    except Exception as e:
        # Handle other possible errors
        #return jsonify({'error': 'An error occurred while generating the image'}), 500
        return render_template('prompting.html', image_url=website_url)

if __name__ == '__main__':
    app.run(debug=True)


"""
Van Krevelen图是在石油化工领域应用广泛的图表，用来描述碳和氢之间的相对含量关系。
"""

import numpy as np
import matplotlib.pyplot as plt

# 生成数据
carbon = np.arange(0, 1.01, 0.01)  # 碳含量
hydrogen = np.arange(0, 1.01, 0.01)  # 氢含量
oxygen = np.arange(0, 1.01, 0.01)  # 氧含量
plt.xlabel('C/H')
plt.ylabel('O/H')

# # 绘制Van Krevelen图
# for x in range(len(oxygen)):
#     plt.plot(carbon/hydrogen, oxygen[x]/hydrogen, color='black', linewidth=0.5)
#
# for y in range(len(carbon)):
#     plt.plot(carbon[y]/hydrogen, oxygen/hydrogen, color='black', linewidth=0.5)

# 绘制Van Krevelen图
X, Y = np.meshgrid(carbon/hydrogen, oxygen/hydrogen)
plt.plot(X.flatten(), Y.flatten(), color='black', linewidth=0.5)

# 在图表上绘制示例数据
example_data = [(0.2, 0.1, 0.7), (0.3, 0.2, 0.5), (0.1, 0.25, 0.65), (0.15, 0.35, 0.5), (0.35, 0.3, 0.35)]
colors = ['r', 'g', 'b', 'c', 'm']
for i in range(len(example_data)):
    plt.scatter(example_data[i][0]/example_data[i][1], example_data[i][2]/example_data[i][1], color=colors[i], s=50)

# 显示图表
plt.xlabel('C/H')
plt.ylabel('O/H')
plt.show()


import socket
import socks
from htun.args import args

def create_socket():
    if args.proxy:
        server_socket = socks.socksocket()
        proto, rest = args.proxy.split('://')
        host, port = rest.split(':')
        port = int(port)
        if proto == "socks4":
            server_socket.set_proxy(socks.SOCKS4, host, port)
        elif proto == "socks5":
            server_socket.set_proxy(socks.SOCKS5, host, port)
        else:
            print("Invalid protocol: %s (must be one of socks4 or socks5)" %
                  proto)
    else:
        server_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)

    server_socket.connect((args.uri["peer_ip"], args.uri["port"]))

    return server_socket

server_socket = create_socket()


import hashlib
import os
import json

def check_sum():
    git_version = input('Git Version: ')
    svn_version = input('SVN Version: ')
    path = input('Folder: ')
    file_json = input('Output JSON Name: ')

    while True:
        check_del = input(f'Deleted Start Path "{path}/"? [y/N]: ')

        if check_del.lower() == 'y' or check_del.lower() == 'n' or check_del.lower() == 'yes' or check_del.lower() == 'no':
            break

    md5_list = {
        "files":[],
        "svn_version": svn_version,
        "git_version": git_version
    }

    print(f'Start Make JSON: "{file_json}.json"')

    for rootdir, dirs, files in os.walk(path):
        for file in files:
            # print (os.path.join(rootdir, file))

            with open(os.path.join(rootdir, file), 'rb') as get_file:
                byte_size = get_file.read()

                get_file.close()

                if check_del.lower() == 'y' or check_del.lower() == 'yes':
                    md5_list['files'].append({
                        "size":len(byte_size),
                        "name":str(os.path.join(rootdir, file)).replace('\\', '/').lstrip(f'{path}/'),
                        "md5":str(hashlib.md5(byte_size).hexdigest())
                    })

                else:
                    md5_list['files'].append({
                        "size":len(byte_size),
                        "name":str(os.path.join(rootdir, file)).replace('\\', '/'),
                        "md5":str(hashlib.md5(byte_size).hexdigest())
                    })

        with open(file_json+'.json', 'w') as json_file:
            json_file.write(json.dumps(md5_list))
            json_file.close()

if __name__ == '__main__':
    check_sum()
    print('Done!')


from fastapi import FastAPI, Request, Response
import uvicorn
from db.session import engine
from db.base import Base
from core.config import Settings
from apis.base import api_router

def create_tables():  # new
    Base.metadata.create_all(bind=engine)

def create_router(app):
    app.include_router(api_router)

def start_application():
    app = FastAPI(title=Settings.PROJECT_TITLE, version=Settings.PROJECT_VERSION,
                  description=Settings.PROJECT_DESCRIPTION)
    create_tables()
    create_router(app)
    return app

app = start_application()

async def catch_exceptions_middleware(request: Request, call_next):
    try:
        return await call_next(request)
    except Exception:
        return Response("Internal server error", status_code=500)

app.middleware('http')(catch_exceptions_middleware)

@app.get("/")
def read_root():
    return {"Hello": "World"}

if __name__ == "__main__":
    uvicorn.run(app, host="0.0.0.0", port=8000)


import sys
import torch as th
import torch.nn.functional as F
from torch import nn
from torch.autograd import Variable
import numpy as np
import math
import data_utility
import pickle
import matplotlib
matplotlib.use('Agg')
import matplotlib.pyplot as plt
import matplotlib.cm as cm
import seaborn as sns
import pandas as pd
sys.setrecursionlimit(50000)
class FeatureExtractionModel(nn.Module):
    def __init__(self, text_emb, rela_emb, bilstm, W, cnn1, cnn2, cnn3):
        super(FeatureExtractionModel, self).__init__()
        self.word_embedding = text_emb
        self.rela_embedding = rela_emb
        # LSTM layer
        self.bilstm = bilstm
        # Attention
        self.W = W
        # CNN layer
        self.cnn_1 = cnn1
        self.cnn_2 = cnn2
        self.cnn_3 = cnn3
        self.activation = nn.ReLU()

    def forward(self, question, word_relation, rela_relation):
        question = th.transpose(question, 0, 1)
        rela_relation = th.transpose(rela_relation, 0, 1)
        word_relation = th.transpose(word_relation, 0, 1)

        question = self.word_embedding(question)
        #question = self.dropout(question)
        rela_relation = self.rela_embedding(rela_relation)
        #rela_relation = self.dropout(rela_relation)
        word_relation = self.word_embedding(word_relation)
        #word_relation = self.dropout(word_relation)
#        self.bilstm.flatten_parameters()
        question_out, _ = self.bilstm(question)
        question_out = question_out.permute(1,2,0)
        #question_out = self.dropout(question_out)
        word_relation_out, word_relation_hidden = self.bilstm(word_relation)
        rela_relation_out, _ = self.bilstm(rela_relation, word_relation_hidden)
        #word_relation_out = self.dropout(word_relation_out)
        #rela_relation_out = self.dropout(rela_relation_out)
        relation = th.cat([rela_relation_out, word_relation_out], 0)
        relation = relation.permute(1,0,2)
        #print('relation', relation)
        # attention layer
        energy = th.matmul(relation, self.W)
        #print('energy', energy)
        energy = th.matmul(energy, question_out)
        #print('energy', energy.shape)
        energy_tmp = energy.view(energy.shape[0], energy.shape[1]*energy.shape[2])
        alpha = F.softmax(energy_tmp, dim=-1)
        alpha = alpha.view(energy.shape[0], energy.shape[1], energy.shape[2])
        return alpha

def extract_relas(tuples):
    pos_relas = []
    neg_relas = []
    for t in tuples:
        if t[2] == 1:
            pos_relas.append('.'.join(t[1] + [t[0]]))
        elif t[2] == 0:
            neg_relas.append('.'.join(t[1] + [t[0]]))
        else:
            print('error'); exit()
    return pos_relas, neg_relas

def to_variable(l):
    v = Variable(th.from_numpy(np.array(l))).cuda()
    return v

import sys
model_path = sys.argv[1]
rela_tokenizer_path = sys.argv[2]
model = th.load(model_path)

layer_list = list(model.children())
print(layer_list)

text_emb = layer_list[1]
rela_emb = layer_list[2]
bilstm = layer_list[3]
cnn1 = layer_list[4]
cnn2 = layer_list[5]
cnn3 = layer_list[6]
W = model.W

feature_extractor = FeatureExtractionModel(text_emb, rela_emb, bilstm, W, cnn1, cnn2, cnn3)
feature_extractor = feature_extractor.eval()
with open('tokenizer/ques_tokneizer_300.pkl', 'rb') as f:
    ques_tokenizer = pickle.load(f)
with open(rela_tokenizer_path, 'rb') as f:
    rela_tokenizer = pickle.load(f)
datas = data_utility.load_test_data()
for index, (i, ques_text, step_list) in enumerate(datas):
    if index != 337:
        continue
    for rela_len in range(0, len(step_list)-1):
        try:
            print('\r', index, end='')
            pos_relas, neg_relas = extract_relas(step_list[rela_len])
            t_ques_text = ques_tokenizer.process([ques_text])
            t_pos_rela_text, t_pos_rela = rela_tokenizer.process(pos_relas)
            #print(t_pos_rela)
            #print(t_pos_rela_text)
            v_ques_text = to_variable(t_ques_text)
            v_pos_rela = to_variable(t_pos_rela)
            v_pos_rela_text = to_variable(t_pos_rela_text)
            #print(v_ques_text)
            #print(v_pos_rela_text)
            #print(v_pos_rela)
            A = feature_extractor(v_ques_text, v_pos_rela_text, v_pos_rela)
            #print(ques_text, pos_relas)
            print(A)
            A = A.cpu().detach().numpy()
            rela_texts = rela_tokenizer.split(pos_relas[0])
            relas = rela_tokenizer.split_rela(pos_relas[0])
            ques_tokens = ques_text.split()
            print(ques_tokens)
            print(rela_texts)
            print(relas)
            if len(ques_tokens) < 5:
                ques_tokens = ['<PAD>'] * (5 - len(ques_tokens)) + ques_tokens
            df = pd.DataFrame( A[0])
            #columns = ques_tokens,
            #                           index = rela_texts + relas)
            ax = sns.heatmap( data= df, linewidth=0.1, cmap='Reds', annot=False,
                    square=True, cbar=False)
            #ax.set_title('TITLE HERE')
            #ax.set_xlabel('x_label')
            #ax.set_ylabel('y_label')
            #ax.xaxis.tick_top()
            ax.xaxis.set_ticks_position('top')

            # labels rotation
            #ax.set_xticklabels(ax.get_xticklabels(), rotation=0)
            #ax.set_yticklabels(ax.get_yticklabels(), rotation=0)

            plt.savefig(f'./{index}_{rela_len}.jpg', dpi=100)
        except ValueError:
            continue


#!/usr/bin/env python
# coding: utf-8

## Code used to obtain search results from the Bing web search API ##
## Last edited by Bronwen Hunter on 05/01/2021 ##

import requests
import pandas as pd
import json
import math
import time
from datetime import datetime
import argparse

class bing_wrapper():

    def __init__(self,
                 queryfile, # File where list of queries is stored
                 api_key,   # API key used to access Bing services
                 directory, # Directory where output files will be stored
                 numberofresults=300,
                 pagesize=100,
                 language='en'):

        df = pd.read_csv(queryfile)
        self.querylist = list(df['queries'])
        self.api_key = api_key
        self.directory = directory
        self.numberofresults = numberofresults
        self.pagesize = pagesize
        self.language = language

    def bing_search(self):

        ''' Function to retrieve all results'''

        search_url = "https://api.bing.microsoft.com/v7.0/search"
        resultsdict = {}
        for query in self.querylist:
            numberofpages = math.ceil(self.numberofresults/self.pagesize)
            urllist = []
            namelist = []
            for i in range(0,numberofpages):
                headers = {"Ocp-Apim-Subscription-Key":self.api_key}
                params = {"q":query,
                          "textDecorations":False,
                          "textFormat":"HTML",
                          "count":self.pagesize, # Returns a maximum of 100 results per page
                          "offset":i*self.pagesize, # Allows for pagination
                          "safeSearch":"strict",
                          "setLang":self.language }
                response = requests.get(search_url, headers=headers, params=params)
                search_results = response.json()
                print("Retrieving page {}/{} for query:{}".format(i+1, numberofpages, query))
                results = search_results.get('webPages', {}).get('value', {})
                urllist = urllist + [results[j]['url'] for j in enumerate(results)]
                namelist = namelist + [results[j]['name'] for j in enumerate(results)]
                time.sleep(5)

            resultsdf = pd.DataFrame() # Creating a dataframe to stored the results
            resultsdf['external_url']=urllist
            resultsdf['title'] = namelist
            resultsdf['query'] = [query for item in urllist]
            resultsdf['source'] = ['Bing' for item in urllist]
            resultsdict[query] = resultsdf
        allresultssdf = bing_wrapper.deduplicated_save(self, resultsdict)
        return allresultssdf

    def deduplicated_save(self, resultsdict):

        '''Function to deduplicate and save bing search results as a CSV file'''

        fullresults = pd.concat(resultsdict.values(), ignore_index=True)
        deduplicated = fullresults.drop_duplicates(subset = 'external_url')   # Deduplicating by result URL
        deduplicated['id'] = [('B'+str(i)) for i in range(len(deduplicated))] # Adding an ID to each result
        now = datetime.now()
        dt_string = now.strftime("%d-%m-%Y")                                  # Adding the date the search was conducted to the filename
        deduplicated.to_csv(self.directory+"bingsearch_allqueries_{}.csv".format(dt_string))
        print("Documents are written to the {folder}".format(folder=self.directory))
        return deduplicated

def main():

    """The main function defines the interface with the users."""

    parser = argparse.ArgumentParser()
    parser.add_argument('--queryfile', default=None, required=True, type=str, help='Filepath where queries are stored')
    parser.add_argument('--api_key', default=None, required=True, type=str, help='API key for bing endpoint')
    parser.add_argument('--directory', default=None, required=True, type=str, help='Directory where output will be saved')
    parser.add_argument('--numberofresults', default=300, required=False, type=int, help='Number of results to return')
    parser.add_argument('--pagesize', default=100, required=False, type=int, help='Number of results returned per page')
    parser.add_argument('--lang', default = 'en', required=False, type=str, help='Language of the search results' )

    args = parser.parse_args()
    print(args)

    inst = bing_wrapper(args.queryfile,
                       args.api_key,
                       args.directory,
                       args.numberofresults,
                       args.pagesize,
                       args.lang)

    inst.bing_search()

if __name__ == "__main__":
    main()


from PySide2.QtWidgets import QWidget, QPushButton, QVBoxLayout, QTabWidget
from PySide2.QtCore import SIGNAL, QObject

from silx.gui.widgets import PeriodicTable

class ElementTable(QWidget):
    """
            Class to handle elements on a spectrum
            Display of rays for each known elements
    """
    def __init__(self, master, window):
        super().__init__(window)
        self.master = master
        self.window = window
        self.set_ui()
        self.window.show()
        self.selected_elements = []

    def set_ui(self):
        """

        set GUI for element table

        :return:
        """
        layout = QVBoxLayout()
        b_pt = QPushButton('Periodic Table')
        QObject.connect(b_pt, SIGNAL('clicked()'), self._start_periodic_table)
        layout.addWidget(b_pt)
        b_pt.show()
        btn_quit = QPushButton('Quit')
        QObject.connect(btn_quit, SIGNAL('clicked()'), self.quit)
        layout.addWidget(btn_quit)
        btn_quit.show()
        self.window.setLayout(layout)

    def _start_periodic_table(self):
        """
        Display periodic table. Cf. doc silx
        :return:
        """
        self.w = QTabWidget()
        self.pt = PeriodicTable.PeriodicTable(self.w)
        self.pt.sigElementClicked.connect(self._my_slot)
        self.pc = PeriodicTable.PeriodicCombo(self.w)
        self.pl = PeriodicTable.PeriodicList(self.w)
        self.pl.sigSelectionChanged.connect(self.change_list)
        self.pc.sigSelectionChanged.connect(self.change_combo)

        comboContainer = QWidget(self.w)
        comboContainer.setLayout(QVBoxLayout())
        comboContainer.layout().addWidget(self.pc)

        self.w.addTab(self.pt, "PeriodicTable")
        self.w.addTab(self.pl, "PeriodicList")
        self.w.addTab(comboContainer, "PeriodicCombo")

        self.pt.show()
        self.pl.show()
        self.pc.show()

        self.w.show()

    def _my_slot(self, item):
        """

            function to toggle clicked element

            :param item:
            :return:
        """
        self.pt.elementToggle(item)
        tmp = list()
        for e in self.pt.getSelection():
            tmp.append(e.symbol)

        if len(tmp) > len(self.selected_elements):
            self.selected_elements = tmp
            self._plot_element()
        else:
            to_erase = list(set(self.selected_elements) - set(tmp))
            self._supress_plot_element(to_erase)
            self.selected_elements = tmp

    def _plot_element(self):
        """

            Plot all line in lineident file for each ray of the selected element

            :return:
        """
        e = self.selected_elements[-1]
        ordre = "ordre_{}".format(int(self.master.num_ordre.text()))
        x_lower = \
            self.master.dict_tabs["Tab_{}".format(self.master.tabs.currentIndex())]["import"].delim["delim_data"][
                ordre][
                'lim_basse']
        x_upper = \
            self.master.dict_tabs["Tab_{}".format(self.master.tabs.currentIndex())]["import"].delim["delim_data"][
                ordre][
                'lim_haute']
        ele_in_lims = []
        key_list = []

        for key in self.master.dict_tabs["Tab_{}".format(self.master.tabs.currentIndex())]["import"].lineident_json[e]:
            if key != 'number':
                if x_lower < self.master.dict_tabs["Tab_{}".format(self.master.tabs.currentIndex())][
                        "import"].lineident_json[e][key]["lambda"] < x_upper:

                    ele_in_lims.append(self.master.dict_tabs["Tab_{}".format(self.master.tabs.currentIndex())][
                                           "import"].lineident_json[e][key]["lambda"])
                    key_list.append(key)

        y = [self.master.dict_tabs["Tab_{}".format(self.master.tabs.currentIndex())]["canvas"].y_lim[0],
             self.master.dict_tabs["Tab_{}".format(self.master.tabs.currentIndex())]["canvas"].y_lim[1]]
        for i in range(len(ele_in_lims)):
            x = [ele_in_lims[i], ele_in_lims[i]]
            self.master.dict_tabs["Tab_{}".format(
                self.master.tabs.currentIndex())]["ax"].plot(x, y, linewidth=2.0, linestyle='-.', label="{}".format(
                key_list[i]), color='b')
            self.master.dict_tabs["Tab_{}".format(
                self.master.tabs.currentIndex())]["ax"].text(ele_in_lims[i], self.master.dict_tabs["Tab_{}".format(
                self.master.tabs.currentIndex())]["canvas"].y_lim[0], "{}".format(e), color='b', fontsize=12)
            self.master.dict_tabs["Tab_{}".format(self.master.tabs.currentIndex())]["fig"].canvas.draw()

    def _supress_plot_element(self, to_erase):

        """

            Supress all ploted line when user unselect an element in the table

            :param to_erase:
            :return:
        """

        k = 0
        l = len(self.master.dict_tabs["Tab_{}".format(self.master.tabs.currentIndex())]["ax"].lines)
        for j in range(l):
            line = self.master.dict_tabs["Tab_{}".format(self.master.tabs.currentIndex())]["ax"].lines[j - k]
            if line.get_label()[0:2].replace(" ", "") == to_erase[0]:
                self.master.dict_tabs["Tab_{}".format(self.master.tabs.currentIndex())]["ax"].lines.remove(line)
                k += 1
        self.master.dict_tabs["Tab_{}".format(self.master.tabs.currentIndex())]["fig"].canvas.draw()

        k = 0
        l = len(self.master.dict_tabs["Tab_{}".format(self.master.tabs.currentIndex())]["ax"].texts)
        for j in range(l):
            txt = self.master.dict_tabs["Tab_{}".format(self.master.tabs.currentIndex())]["ax"].texts[j - k]
            if txt.get_text() == to_erase[0]:
                self.master.dict_tabs["Tab_{}".format(self.master.tabs.currentIndex())]["ax"].texts.remove(txt)
                k += 1
        self.master.dict_tabs["Tab_{}".format(self.master.tabs.currentIndex())]["fig"].canvas.draw()

    def change_list(self, items):
        print("New list selection:", [item.symbol for item in items])

    def change_combo(self, item):
        print("New combo selection:", item.symbol)

    def click_table(self, item):
        print("New table click: %s (%s)" % (item.name, item.subcategory))

    def quit(self):
        self.window.close()