"""
This spider is a MentoringXML spider created on top of the XMLSpider
scrapy crawl mentoring_xml -a extract=1 -a url="https://connect.mentoring.org/admin/index.php?action=linkedIn"

sample url:
    https://connect.mentoring.org/admin/index.php?action=linkedIn
"""

from brightcorp.base.xmlspider import XMLSpider
from brightcorp.items import BrightcorpItemLoader
from brightcorp.processors import NormalizedJoin, Prefix, Strip

class MentoringXML(XMLSpider):

    follow_job_url = False
    name = 'mentoring_xml'
    tag = 'job'

    field_xpaths = {
        'title': '//JobTitle/text()',
        'company': '//company/text()',
        'zip_code': '//PostalCode/text()',
        'jobcategory': '//JobFunction/text()',
        'description': '//JobDescription/node()',
        'company_description': '//CompanyDescription/node()'
    }

    location_xpaths = ['//city/text()', '//state/text()']
    refnum_xpath = ['//JobId/text()']

    def parse_job(self, response):
        loader = BrightcorpItemLoader(response=response)

        for k, v in self.field_xpaths.iteritems():
            loader.add_xpath(k, v)

        loader.add_xpath(
            'location', self.location_xpaths, NormalizedJoin(", ")
            )
        loader.add_xpath(
            'referencenumber', self.refnum_xpath, Prefix('%s-' % self.name)
            )
        loader.add_xpath(
            'url', self.refnum_xpath, Strip(),
            Prefix('https://connect.mentoring.org/index.php?section=overview&action=view&fwID=')
            )
        yield loader.load_item()


from sqlalchemy import Column, Integer, String, Float, ForeignKey, Table, DateTime, create_engine, MetaData
from sqlalchemy.orm import relationship, backref
from sqlalchemy.ext.declarative import declarative_base
import datetime

engine = create_engine('sqlite:////tmp/test.db', convert_unicode=True)
metadata = MetaData(bind=engine)

Base = declarative_base()

vehicles = Table(
    "vehicles",
    Base.metadata,
    Column("id", String, primary_key=True),
    Column("no_of_seats", Integer),
    Column("engine", Float),
    Column("fuel_type", String),
    Column("gear_type", String),
    Column("company", String),
    Column('model', String),
    Column('build_year', Integer)
)

stations = Table(
    "stations",
    Base.metadata,
    Column("code", String, primary_key=True),
    Column("name", String),
    Column("address_id", Integer, ForeignKey("addresses.id")),
    Column("email", String),
    Column("phone", Integer)
)

addresses = Table(
    "addresses",
    Base.metadata,
    Column("id", Integer, primary_key=True),
    Column('street_no', String),
    Column("street_name", String),
    Column("city", String),
    Column("postcode", String),
    Column("country", String)
)

customers = Table(
    "customers",
    Base.metadata,
    Column("username", String, primary_key=True),
    Column("name", String),
    Column("address_id", Integer, ForeignKey("addresses.id")),
    Column("email", String),
    Column("phone", Integer),
    Column("type", String),
    Column("password", String)
)

bookings = Table(
    "bookings",
    Base.metadata,
    Column("id", String, primary_key=True),
    Column("car_id", String, ForeignKey("vehicles.id")),
    Column("customer_uname", String, ForeignKey("customers.username")),
    Column("booked_on", DateTime, default=datetime.datetime.utcnow),
    Column("booked_for", DateTime),
    Column("station_code", String, ForeignKey("stations.code")),
    Column("return_on", DateTime)
)

trips = Table(
    "trips",
    Base.metadata,
    Column("id", Integer, primary_key=True),
    Column("booking_id", String, ForeignKey("bookings.id")),
    Column("distance", Float),
    Column("duration", Float),
    Column("cost", Float),
    Column("status", String)
)

payments = Table(
    "payments",
    Base.metadata,
    Column("id", Integer, primary_key=True),
    Column("trip_id", Integer, ForeignKey("trips.id")),
    Column("method", String),
    Column("status", String)
)


import cv2

cap = cv2.VideoCapture('C:/Users/QIU/Desktop/Code/CircleDetect/data/video/moulde1.mp4')
frame_count = 217
while True:
    ret, frame = cap.read()
    if ret is not True:
        print("That's all!")
        exit()
    else:
        frame_name = 'C:/Users/QIU/Desktop/Code/CircleDetect/data/dataset' + '/' + str(frame_count) + '.jpg'
        cv2.imwrite(frame_name, frame)
        frame_count += 1
        if cv2.waitKey(3) & 0xFF == ord('q'):
            break
# When everything done, release the capture
cap.release()
cv2.destroyAllWindows()


# Invert a binary tree.

"""
Definition of TreeNode:
class TreeNode:
    def __init__(self, val):
        self.val = val
        self.left, self.right = None, None
"""
class Solution:
    # @param root: a TreeNode, the root of the binary tree
    # @return: nothing
    def invertBinaryTree(self, root):
        # write your code here
        # if root==None:
        #     return None
        # st=[root]
        # while len(st)>0:
        #     node=st.pop()
        #     node.left, node.right=node.right, node.left
        #     if node.right:
        #         st.append(node.right)
        #     if node.left:
        #         st.append(node.left)
        # return root

        def invertsub(root):
            if root==None:
                return None
            root.left, root.right=root.right, root.left
            invertsub(root.left)
            invertsub(root.right)
        invertsub(root)
        return root


#!/usr/bin/python
# Script to plot the c++ output data
# Daniel Crews 3/16/2017

from pylab import * # For plotting and math
import os # For input/output

mi = 129
mj = 64

##### Import Data
## File names
xname = 'converged_grid_x.txt'
yname = 'converged_grid_y.txt'
xcenter_name = 'cell_x.dat'
ycenter_name = 'cell_y.dat'

#xnormname = 'dsj_x.dat'
#ynormname = 'dsj_y.dat'
xnormname = 'dsi_x.dat'
ynormname = 'dsi_y.dat'

##### Import data
## GRID
gridx = loadtxt(xname, skiprows=0)
gridy = loadtxt(yname, skiprows=0)
dataxc = hstack((gridx, gridx[:, 0].reshape(-1, 1)))
datayc = hstack((gridy, gridy[:, 0].reshape(-1, 1)))

## NORMALS
xnorm = loadtxt(xnormname, skiprows=0)
ynorm = loadtxt(ynormname, skiprows=0)

## CENTERS
cellx = loadtxt(xcenter_name, skiprows=0)
celly = loadtxt(ycenter_name, skiprows=0)

#print(xnorm.shape)
#print(dataxc.shape)

## Reshape
xnorm = reshape(xnorm, (65, 128), order='F')
ynorm = reshape(ynorm, (65, 128), order='F')

cellx = reshape(cellx, (64, 128), order='F')
celly = reshape(celly, (64, 128), order='F')

#xnorm = reshape(xnorm, (64, 129), order='F')
#ynorm = reshape(ynorm, (64, 129), order='F')

halfgridx = zeros(dataxc.shape)
halfgridy = zeros(dataxc.shape)

#print(halfgridx.shape)
#print(xnorm.shape)

##### Calculate x and y positions of normal
for i in range(mj):
    halfgridx[i, :] = dataxc[i, :] + (dataxc[i+1, :] - dataxc[i, :])/2.
    halfgridy[i, :] = datayc[i, :] + (datayc[i+1, :] - datayc[i, :])/2.

for j in range(mi):
    halfgridx[:, j] = dataxc[:, j] + (dataxc[:, j+1] - dataxc[:, j])/2.
    halfgridy[:, j] = datayc[:, j] + (datayc[:, j+1] - datayc[:, j])/2.

#print(halfgridx[0:65][0])
#print(dataxc[0:65][0])

## Plot the grid
figure()
for i in range(mj+1):
    plot(dataxc[i, :], datayc[i, :], color='k')

for i in range(mi):
    plot(dataxc[:, i], datayc[:, i], color='k')

for i in range(127):
    scatter(cellx[:, i], celly[:, i], color='g')

for i in range(65):
    for j in range(128):
        arrow(halfgridx[i, j], halfgridy[i, j],
              xnorm[i, j], ynorm[i, j], fc="k", ec="k",
              head_width=0.0005, head_length=0.0001)

xlabel('Position x')
ylabel('Position y')
axis('equal')

#figure()
#for i in range(mj+1):
#    plot(halfgridx[i, :], halfgridy[i, :], color='k')

show()


# Copyright 2019 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""BERT model input pipelines."""

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import os
import json

import tensorflow as tf

from data import bert_tokenization
from data.bert_regression_data_lib import file_based_convert_examples_to_features, StsProcessor

class BertRegressionDataset(object):
  def __init__(self, data_dir):
    self.data_dir = data_dir
    self.metadata = None

  def get_meta_data(self):
    if not self.metadata:
      with tf.io.gfile.GFile(self._get_metadata_path(), 'rb') as reader:
        self.meta_data = json.loads(reader.read().decode('utf-8'))
    return self.meta_data

  def get_dataset(self, set_type, batch_size, is_training=None):
    self.metadata = self.get_meta_data()
    if is_training is None:
      is_training = set_type == 'train'
    return create_regression_dataset(
      self._get_tf_record_path(set_type),
      seq_length=self.metadata['max_seq_length'],
      batch_size=batch_size,
      is_training=is_training,
      drop_remainder=is_training)

  def gen_tf_records(self, processor, vocab_file, max_seq_length,
                     do_lower_case=True, set_types=None):
    tokenizer = bert_tokenization.FullTokenizer(
      vocab_file=vocab_file, do_lower_case=do_lower_case)

    meta_data = {
      "max_seq_length": max_seq_length,
    }

    if set_types is None:
      set_types = processor.get_set_types()

    for set_type in set_types:
      tf_record_path = self._get_tf_record_path(set_type)
      input_data_examples = processor.get_examples(self.data_dir, set_type)
      file_based_convert_examples_to_features(input_data_examples,
                                              max_seq_length, tokenizer,
                                              tf_record_path)
      meta_data['{}_data_size'.format(set_type)] = len(input_data_examples)
      if set_type == 'dev':
        meta_data['eval_data_size'.format(set_type)] = len(input_data_examples)

    with tf.io.gfile.GFile(self._get_metadata_path(), "w") as writer:
      writer.write(json.dumps(meta_data, indent=4) + "\n")

  def _get_tf_record_path(self, set_type):
    return os.path.join(self.data_dir, '{}.tf_record'.format(set_type))

  def _get_metadata_path(self):
    return os.path.join(self.data_dir, 'meta_data')

def create_regression_dataset(file_path,
                              seq_length,
                              batch_size,
                              is_training=True,
                              drop_remainder=True):
  """Creates input dataset from (tf)records files for train/eval."""
  name_to_features = {
      'input_ids': tf.io.FixedLenFeature([seq_length], tf.int64),
      'input_mask': tf.io.FixedLenFeature([seq_length], tf.int64),
      'segment_ids': tf.io.FixedLenFeature([seq_length], tf.int64),
      'vals': tf.io.FixedLenFeature([], tf.float32),
      'is_real_example': tf.io.FixedLenFeature([], tf.int64),
  }
  input_fn = _file_based_input_fn_builder(file_path, name_to_features)
  dataset = input_fn()

  def _select_data_from_record(record):
    x = {
        'input_word_ids': record['input_ids'],
        'input_mask': record['input_mask'],
        'input_type_ids': record['segment_ids']
    }
    y = record['vals']
    return (x, y)

  dataset = dataset.map(_select_data_from_record)

  if is_training:
    dataset = dataset.shuffle(100)
    dataset = dataset.repeat()

  dataset = dataset.batch(batch_size, drop_remainder=drop_remainder)
  dataset = dataset.prefetch(1024)
  return dataset

def _decode_record(record, name_to_features):
  """Decodes a record to a TensorFlow example."""
  example = tf.io.parse_single_example(record, name_to_features)

  # tf.Example only supports tf.int64, but the TPU only supports tf.int32.
  # So cast all int64 to int32.
  for name in list(example.keys()):
    t = example[name]
    if t.dtype == tf.int64:
      t = tf.cast(t, tf.int32)
    example[name] = t

  return example

def _file_based_input_fn_builder(input_file, name_to_features):
  """Creates an `input_fn` closure to be passed for BERT custom training."""

  def input_fn():
    """Returns dataset for training/evaluation."""
    # For training, we want a lot of parallel reading and shuffling.
    # For eval, we want no shuffling and parallel reading doesn't matter.
    d = tf.data.TFRecordDataset(input_file)
    d = d.map(lambda record: _decode_record(record, name_to_features))

    # When `input_file` is a path to a single file or a list
    # containing a single path, disable auto sharding so that
    # same input file is sent to all workers.
    if isinstance(input_file, str) or len(input_file) == 1:
      options = tf.data.Options()
      #options.experimental_distribute.auto_shard_policy = (
      #    tf.data.experimental.AutoShardPolicy.OFF)
      d = d.with_options(options)
    return d

  return input_fn


import pymatgen as mg
import numpy as np
import random
import math
import os
from pymatgen.analysis.diffraction import xrd

class StrainGen(object):
    """
    Class used to apply stochastic, symmetry-preserving sets of
    strain to a pymatgen structure object.
    """

    def __init__(self, struc, max_strain=0.04):
        """
        Args:
            struc: pymatgen structure object
            max_strain: maximum allowed change in the magnitude
                of the strain tensor components
        """
        self.calculator = xrd.XRDCalculator()
        self.struc = struc
        self.max_strain = max_strain

    @property
    def sg(self):
        return self.struc.get_space_group_info()[1]

    @property
    def conv_struc(self):
        sga = mg.symmetry.analyzer.SpacegroupAnalyzer(struc)
        return sga.get_conventional_standard_structure()

    @property
    def lattice(self):
        return self.struc.lattice

    @property
    def matrix(self):
        return self.struc.lattice.matrix

    @property
    def diag_range(self):
        max_strain = self.max_strain
        return np.linspace(1-max_strain, 1+max_strain, 1000)

    @property
    def off_diag_range(self):
        max_strain = self.max_strain
        return np.linspace(0-max_strain, 0+max_strain, 1000)

    @property
    def sg_class(self):
        sg = self.sg
        if sg in list(range(195, 231)):
            return 'cubic'
        elif sg in list(range(16, 76)):
            return 'orthorhombic'
        elif sg in list(range(3, 16)):
            return 'monoclinic'
        elif sg in list(range(1, 3)):
            return 'triclinic'
        elif sg in list(range(76, 195)):
            if sg in list(range(75, 83)) + list(range(143, 149)) + list(range(168, 175)):
                return 'low-sym hexagonal/tetragonal'
            else:
                return 'high-sym hexagonal/tetragonal'

    @property
    def strain_tensor(self):
        diag_range = self.diag_range
        off_diag_range = self.off_diag_range
        s11, s22, s33 = [random.choice(diag_range) for v in range(3)]
        s12, s13, s21, s23, s31, s32 = [random.choice(off_diag_range) for v in range(6)]
        sg_class = self.sg_class

        if sg_class in ['cubic', 'orthorhombic', 'monoclinic', 'high-sym hexagonal/tetragonal']:
            v1 = [s11, 0, 0]
        elif sg_class == 'low-sym hexagonal/tetragonal':
            v1 = [s11, s12, 0]
        elif sg_class == 'triclinic':
            v1 = [s11, s12, s13]

        if sg_class in ['cubic', 'high-sym hexagonal/tetragonal']:
            v2 = [0, s11, 0]
        elif sg_class == 'orthorhombic':
            v2 = [0, s22, 0]
        elif sg_class == 'monoclinic':
            v2 = [0, s22, s23]
        elif sg_class == 'low-sym hexagonal/tetragonal':
            v2 = [-s12, s22, 0]
        elif sg_class == 'triclinic':
            v2 = [s21, s22, s23]

        if sg_class == 'cubic':
            v3 = [0, 0, s11]
        elif sg_class == 'high-sym hexagonal/tetragonal':
            v3 = [0, 0, s33]
        elif sg_class == 'orthorhombic':
            v3 = [0, 0, s33]
        elif sg_class == 'monoclinic':
            v3 = [0, s23, s33]
        elif sg_class == 'low-sym hexagonal/tetragonal':
            v3 = [0, 0, s33]
        elif sg_class == 'triclinic':
            v3 = [s31, s32, s33]

        return np.array([v1, v2, v3])

    @property
    def strained_matrix(self):
        return np.matmul(self.matrix, self.strain_tensor)

    @property
    def strained_lattice(self):
        return mg.Lattice(self.strained_matrix)

    @property
    def strained_struc(self):
        new_struc = self.struc.copy()
        new_struc.lattice = self.strained_lattice
        return new_struc

    @property
    def strained_spectrum(self):
        struc = self.strained_struc
        pattern = self.calculator.get_pattern(struc, two_theta_range=(0,80))
        angles, intensities = pattern.x, pattern.y

        x = np.linspace(10, 80, 4501)
        y = []

        step_size = (80. - 10.)/4501.
        half_step = step_size/2.0
        for val in x:
            ysum = 0
            for (ang, pk) in zip(angles, intensities):
                if np.isclose(ang, val, atol=half_step):
                    ysum += pk
            y.append(ysum)

        conv = []
        for (ang, int) in zip(x, y):
            if int != 0:
                gauss = [int*np.exp((-(val - ang)**2)/0.15) for val in x]
                conv.append(gauss)

        mixed_data = zip(*conv)
        all_I = []
        for values in mixed_data:
            noise = random.choice(np.linspace(-0.75, 0.75, 1000))
            all_I.append(sum(values) + noise)

        shifted_vals = np.array(all_I) - min(all_I)
        scaled_vals = 100*np.array(shifted_vals)/max(shifted_vals)
        all_I = [[val] for val in scaled_vals]

        return all_I

def main(struc, num_strains, max_strain):

    strain_generator = StrainGen(struc, max_strain)

    strained_patterns = [strain_generator.strained_spectrum for i in range(num_strains)]

    return strained_patterns


import requests
import json

headers = {
    "Referer": "https://m.douban.com/tv/american",
    "User-Agent": "Mozilla/5.0 (Linux; Android 8.0.0; Pixel 2 XL Build/OPD1.170816.004) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.87 Mobile Safari/537.36"
}
request_url = "https://m.douban.com/rexxar/api/v2/subject_collection/filter_tv_american_hot/items?start={}&count=18&loc_id=108288"
# i=0
# url_list = []
# while i < 1084 :
#     url_list.append(request_url.format(i))
#     i += 18
# for url in url_list:
#     response = requests.get(url, headers=headers)
#     json_str = json.loads(response.content.decode("utf-8"))
#     print(json_str)
    # with open("american.json", "a+", encoding="utf-8") as f:
    #     ret = json.d
    #     f.write(ret)
# response = requests.get(request_url.format(0), headers=headers)
# json_str = json.loads(response.content.decode("utf-8"))
# # print(json_str)
# content = json_str["subject_collection_items"]
# print(content)

class DoubanSpider():
    def __init__(self):
        self.headers = {
            "Referer": "https://m.douban.com/tv/american",
            "User-Agent": "Mozilla/5.0 (Linux; Android 8.0.0; Pixel 2 XL Build/OPD1.170816.004) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.87 Mobile Safari/537.36"
        }
        # self.url = "https://m.douban.com/rexxar/api/v2/subject_collection/filter_tv_american_hot/items?start={}&count=18&loc_id=108288"
        self.url_list = [
            {
                "url": "https://m.douban.com/rexxar/api/v2/subject_collection/filter_tv_american_hot/items?start={}&count=18&loc_id=108288",
                "country":"america"
            },
            {
                "url":"https://m.douban.com/rexxar/api/v2/subject_collection/filter_tv_english_hot/items?start=0&count=18&loc_id=108288",
                "country":"british"
            },
            {
                "url":"https://m.douban.com/rexxar/api/v2/subject_collection/filter_tv_domestic_hot/items?start=0&count=18&loc_id=108288",
                "country":"domestic"
            }
        ]

    def parse_url(self, url):
        response = requests.get(url, headers=self.headers)
        return response.content.decode("utf-8")

    def get_content_dict(self, json_str):
        content_dict = json.loads(json_str)
        return content_dict["subject_collection_items"], content_dict["total"]

    def save_content(self, content_list):
        with open("douban2.txt", "a", encoding="utf-8") as f:
            # f.write(country)
            for content in content_list:
                tv_info = json.dumps(content, ensure_ascii=False)
                # 将提取到的数据写入文件
                f.write(tv_info)
                f.write("\n")

    def run(self):
        # 组织url地址
        for url_dict in self.url_list:
            country_list = [url_dict["country"],]
            self.save_content(country_list)
            url_temp = url_dict["url"]
            i = 0
            count = 100
            while i < count:
                url = url_temp.format(i)
                i += 18
            # 获取响应内容json_str
                json_str = self.parse_url(url)
            # 提取数据
                content_list, count = self.get_content_dict(json_str)
                self.save_content(content_list)

if __name__ == '__main__':
    douban_spider = DoubanSpider()
    douban_spider.run()


import os
import re
import json
import logging

import pandas as pd

logger = logging.getLogger(__name__)

pd.set_option('mode.chained_assignment',  None)

def preprocess(root):
    RAW_TRAINING = 'training.1600000.processed.noemoticon.csv'
    RAW_TEST  = 'testdata.manual.2009.06.14.csv'

    def _get_glove_vocab(path):
        # read GloVe embeddings (300-dim)
        lines = []
        with open(os.path.join(path, 'raw', 'glove.6B.300d.txt'), 'r') as file:
            lines = file.readlines()

        # process embeddings
        lines = [l.split() for l in lines]
        vocab = [l[0] for l in lines]
        return vocab

    def _combine_data(path):
        raw_train = pd.read_csv(
            os.path.join(path, RAW_TRAINING),
            encoding='ISO-8859-1',
            header=None,
            names=['target', 'id', 'date', 'flag', 'user', 'text'],
            usecols=['target', 'user', 'text'],
            index_col='user'
        )
        raw_test = pd.read_csv(
            os.path.join(path, RAW_TEST),
            encoding='ISO-8859-1',
            header=None,
            names=['target', 'id', 'date', 'flag', 'user', 'text'],
            usecols=['target', 'user', 'text'],
            index_col='user'
        )
        raw_all = pd.concat([raw_train, raw_test]).sort_index(kind='mergesort')
        return raw_all

    def _convert_to_json(path, raw_all, vocab):
        def _get_words_indices(vocab):
            """Get word indices using pre-defined GloVe.
            """
            indices = {}
            for i in range(len(vocab)):
                indices[vocab[i]] = i
            vocab = {w: i for i, w in enumerate(vocab)}
            return indices

        def _split_line(line):
            """Split given line/phrase into list of words
            """
            return re.findall(r"[\w']+|[.,!?;]", line)

        def _line_to_indices(line, word2id, max_words=25):
            """Converts given phrase into list of word indices.

            - If the phrase has more than `max_words` words,
            returns a list containing indices of the first `max_words` words.

            - If the phrase has less than `max_words` words,
            repeatedly appends integer representing padding index to returned list
            until the list's length is `max_words`.

            Args:
                line: string representing phrase/sequence of words
                word2id: dictionary with string words as keys and int indices as values
                max_words: maximum number of word indices in returned list

            Returns:
                indices: list of word indices, one index for each word in phrase
            """
            pad_id = 0
            line_list = _split_line(line) # split phrase in words
            indices = [word2id[w] if w in word2id else pad_id for w in line_list[:max_words]]
            indices += [pad_id] * (max_words - len(indices))
            return indices

        # get index dictionry of each word
        indices = _get_words_indices(vocab)

        # convert user ID into digits
        user_id_map = {str_id: int_id for int_id, str_id in enumerate(raw_all.index.unique().tolist())}
        curr_ids = raw_all.index.to_series()
        raw_all.index = curr_ids.map(user_id_map)

        # refine raw data
        raw_all = raw_all[raw_all['target'] != 2]
        raw_all.loc[:, 'target'].replace({4: 1}, inplace=True)
        raw_all.loc[:, 'text'] = raw_all['text'].apply(lambda x: _line_to_indices(x, indices))
        raw_all = raw_all.reset_index().groupby('user').agg({'text': lambda x: [i for i in x], 'target': lambda y: [l for l in y]}).rename(columns={'text': 'x', 'target': 'y'})
        raw_all.index = raw_all.index.astype(str)

        # get required elements
        users = raw_all.index.tolist()
        num_samples = raw_all.reset_index().groupby('user').apply(len).values.tolist()
        user_data = raw_all.T.to_dict()

        # create json file
        all_data = {}
        all_data['users'] = users
        all_data['num_samples'] = num_samples
        all_data['user_data'] = user_data

        # save file
        with open(os.path.join(path, 'all_data', 'all_data.json'), 'w') as outfile:
            json.dump(all_data, outfile)

    # set path
    DATASET_NAME = __file__.split('/')[-1].split('.')[0]
    path = os.path.join(os.path.expanduser(root), DATASET_NAME)

    # check if preprocessing has already done
    if not os.path.exists(os.path.join(path, 'all_data')):
        os.makedirs(os.path.join(path, 'all_data'))
    else:
        return

    # make path for GloVe vocabulary
    if not os.path.exists(os.path.join(path, 'vocab')):
        os.makedirs(os.path.join(path, 'vocab'))

    # get GloVe vocabulary
    logger.info(f'[LOAD] [LEAF - {DATASET_NAME.upper()}] Process GloVe embeddings (300 dim.)...!')
    glove_vocab = _get_glove_vocab(path)
    logger.info(f'[LOAD] [LEAF - {DATASET_NAME.upper()}] ...finished processing GloVe embeddings!')

    # combine raw data
    logger.info(f'[LOAD] [LEAF - {DATASET_NAME.upper()}] Combine raw data...!')
    raw_all = _combine_data(os.path.join(path, 'raw'))
    logger.info(f'[LOAD] [LEAF - {DATASET_NAME.upper()}] ...finished combining raw data!')

    # convert to json format
    logger.info(f'[LOAD] [LEAF - {DATASET_NAME.upper()}] Convert data to json format... (this may take several minutes)!')
    _convert_to_json(path, raw_all, glove_vocab)
    logger.info(f'[LOAD] [LEAF - {DATASET_NAME.upper()}] ...finished converting data to json format!')


#!/usr/bin/env python
# coding=utf-8
#
# Author: Lucas
# Date: 2019-07-09 00:07:11

import string

class Solution:
    def longestPalindrome(self, s: str) -> int:
        length = 0
        single = False
        for c in string.ascii_letters:
            count = s.count(c)
            length += count - count % 2
            if count % 2 == 1:
                single = True
        if single:
            length += 1
        return length


from rest_framework import status
from rest_framework.decorators import api_view, authentication_classes, permission_classes
from rest_framework.authentication import BasicAuthentication
from rest_framework.permissions import IsAuthenticated
from rest_framework.response import Response

from student import models
from student import serializers

@api_view(['GET', 'POST', 'PUT', 'PATCH', 'DELETE'])
@authentication_classes([BasicAuthentication])
@permission_classes([IsAuthenticated])
def student_api(request, pk=None):
    if request.method == 'GET':
        # id = request.data.get('id')
        id = pk
        if id is not None:
            student = models.Student.objects.get(id=id)
            serializer = serializers.StudentSerializer(student)
            return Response(serializer.data)

        student = models.Student.objects.all()
        serializer = serializers.StudentSerializer(student, many=True)
        return Response(serializer.data)

    if request.method == 'POST':
        serializer = serializers.StudentSerializer(data=request.data)
        if serializer.is_valid():
            serializer.save()
            return Response({'msg': 'Student Created'}, status=status.HTTP_201_CREATED)
        return Response(serializer.errors, status=status.HTTP_400_BAD_REQUEST)

    if request.method == 'PUT':
        # id = request.data.get('id')
        id = pk
        student = models.Student.objects.get(id=id)
        serializer = serializers.StudentSerializer(student, data=request.data)
        if serializer.is_valid():
            serializer.save()
            return Response({'msg': 'Complete Student Updated'})
        return Response(serializer.errors, status=status.HTTP_400_BAD_REQUEST)

    if request.method == 'PATCH':
        # id = request.data.get('id')
        id = pk
        student = models.Student.objects.get(id=id)
        serializer = serializers.StudentSerializer(student, data=request.data, partial=True)
        if serializer.is_valid():
            serializer.save()
            return Response({'msg': 'Partial Student Updated'})
        return Response(serializer.errors)

    if request.method == 'DELETE':
        # id = request.data.get('id')
        id = pk
        student = models.Student.objects.get(id=id)
        student.delete()
        return Response({'msg': 'Student Deleted'})


import requests
from JavPy.utils.requester import executor, map_f, PlaceHolder
from JavPy.utils.config import proxy

def test(url):
    return requests.head(url, proxies=proxy).status_code == 200

def ping(url, n=5):
    latencies = list(map(lambda rsp: rsp.elapsed.microseconds, filter(
        lambda rsp: rsp and rsp.status_code == 200, executor.map(
            map_f(requests.head, PlaceHolder, proxies=proxy), (url for _ in range(n))
        )
    )))
    return 0 if not latencies else sum(latencies) / len(latencies)

if __name__ == "__main__":
    print(ping("https://playfinder.xyz/v/7j6mgsgxe601k3g#poster=https://findercdn.me/files/sksk-023.jpg"))
    print(ping("https://www.google.com"))


from wheel import WheelPlot, Wheel
import numpy as np
from matplotlib import animation
from matplotlib import pyplot as plt
import math

# use the default wheel for the plot
crossedWheel = Wheel(spokePattern='2x')
w = WheelPlot(crossedWheel)

# rotation to 45 degrees
frameNumber = 200
angleArray1 = np.linspace(0, math.pi, frameNumber/4)
angleArray2 = np.linspace(math.pi,math.pi/2, frameNumber/4)
angleArray3 = np.linspace(math.pi/2, 0, frameNumber/4)
angleArray4 = np.linspace(0, math.pi, frameNumber/4)
angleArray = np.concatenate((angleArray1,angleArray2,angleArray3,angleArray4),axis=1)

def animate(i):
  w.plot(angleArray[i])

anim = animation.FuncAnimation(w.getFigureHandle(), animate, frames=frameNumber, interval=10, blit=True)

anim.save('wheel_animation.mp4', fps=30, extra_args=['-vcodec', 'libx264'])

plt.show()


def fib(n):
    if(n==0):
        return 0
    elif(n<2):
        return 1
    else:
        return fib(n-1)+fib(n-2)
n=int(input("enter the no.of terms"))
for i in range(n):
    print("fibonacci series",fib(i))


from get_data import get_data
import tensorflow as tf
import tensorflow_hub as hub
import fire
from sklearn.metrics.pairwise import cosine_similarity
import time

BATCH_SIZE = 32
MAX_SEQ_LENGTH = 128

def get_elmo_embedding(list_sentence):
    """ Returns embedding for documents """
    list_sentence = list(map(lambda x: ' '.join(x.split()[:MAX_SEQ_LENGTH]), list_sentence))
    embedding_array = []
    with tf.Graph().as_default():
        elmo = hub.Module("https://tfhub.dev/google/elmo/2")
        with tf.Session() as sess:
            sess.run(tf.global_variables_initializer())
            sess.run(tf.tables_initializer())

            total_num_documents = len(list_sentence)
            for i in range(total_num_documents//BATCH_SIZE + 1):
                start = time.time()
                try:
                    batch_sentence = list_sentence[i*BATCH_SIZE:i*BATCH_SIZE + BATCH_SIZE]
                except IndexError: # for last batch
                    batch_sentence = list_sentence[i*BATCH_SIZE:]
                embeddings = elmo(batch_sentence, signature="default", as_dict=True)["default"]

                batch_embeddings = sess.run(embeddings)

                embedding_array.append(batch_embeddings)
                end = time.time()
                print(batch_embeddings.shape)
                print(f"batch {i}, time taken: {end-start}")

    embedding_matrix = np.concatenate(embedding_array, 0)

    return embedding_matrix

def main(domain):
    docs = get_data(domain)
    texts = list(docs.values())
    embedding_matrix = get_elmo_embedding(texts)

    print(embedding_matrix.shape)
    sim = cosine_similarity(embedding_matrix)

    print(sim)

if __name__ == '__main__':
    fire.Fire(main)


from PyQt5.QtWidgets import (QPushButton, QWidget, QGridLayout, QLabel, QVBoxLayout,
                             QHBoxLayout, QScrollArea, QMessageBox)
from PyQt5.QtCore import Qt
from PyQt5.QtGui import QPixmap, QIcon
import ProductUi, CartUi
from sellersigninUi import selelr_signin
from sellerloginUi import seller_login
from UsersigninUi import user_signin
from UserloginUi import login

class ProductWidget(QWidget):
    def __init__(self, parent, product, user):
        super().__init__()
        self.parent = parent
        self.product = product
        self.user = user
        self.setMouseTracking(True)
        layout = QVBoxLayout()
        self.setLayout(layout)
        self.setStyleSheet("background-color:rgb(186,217,189)")

        image_label = QLabel()
        image_label.setPixmap(QPixmap(product.image).scaled(180,180))
        name_label = QLabel(f'
{product.name}
')
        name_label.setAlignment(Qt.AlignCenter)
        layout.addWidget(image_label)
        layout.addWidget(name_label)
        layout.setSpacing(0)

    def enterEvent(self, event):
        self.setStyleSheet("background-color:rgb(171,202,174)")
        super().enterEvent(event)

    def leaveEvent(self, event):
        self.setStyleSheet("background-color:rgb(186,217,189)")
        super().leaveEvent(event)

    def mouseReleaseEvent(self, event):
        if self.user:
            page = ProductUi.MainWidget(self.product, self.user)
            self.parent.goto_page(page, [ProductUi.MainWidget, self.product, self.user])
        else:
            error_message = QMessageBox(self)
            error_message.setIcon(QMessageBox.Critical)
            error_message.setWindowTitle("ERROR")
            error_message.setText("You should be signed-in in order to view products!")
            error_message.setStyleSheet("background-color:white")
            error_message.exec()
        super().mouseReleaseEvent(event)

class MainWidget(QWidget):
    def __init__(self, parent, store, user=None):
        super().__init__()
        self.parent = parent
        self.store = store
        self.user = user
        self.parent.change_toolbar_title(store.name)

        bar_layout = QHBoxLayout()
        cart_pic = QPixmap("./DATABASE/Icons/cart.png")
        cart_button = QPushButton()
        cart_button.setIcon(QIcon(cart_pic))
        cart_button.setIconSize(cart_pic.rect().size()/7)
        cart_button.clicked.connect(self.open_cart)
        bar_layout.addWidget(cart_button, stretch=1)
        spacer = QLabel('')
        bar_layout.addWidget(spacer, stretch=9)

        if user:
            welcome_label = QLabel(f'
Welcome {user.name}!
')
            bar_layout.addWidget(welcome_label, stretch=7)
            quit_button = QPushButton("Log Out of Account")
            quit_button.setStyleSheet("background-color:rgb(32,154,26);font:17px;")
            bar_layout.addWidget(quit_button, stretch=3)
            quit_button.clicked.connect(self.quit)
        else:
            seller_sign_in_button = QPushButton("Sellers\nSign-in")
            seller_sign_in_button.setStyleSheet("background-color:rgb(32,154,26);font:17px;")
            bar_layout.addWidget(seller_sign_in_button, stretch=1)
            seller_sign_in_button.clicked.connect(self.seller_sign_in)

            seller_sign_up_button = QPushButton("Sellers\nSign-up")
            seller_sign_up_button.setStyleSheet("background-color:rgb(32,154,26);font:17px;")
            bar_layout.addWidget(seller_sign_up_button, stretch=1)
            seller_sign_up_button.clicked.connect(self.seller_sign_up)

            user_sign_in_button = QPushButton("Users\nSign-in")
            user_sign_in_button.setStyleSheet("background-color:rgb(32,154,26);font:17px;")
            bar_layout.addWidget(user_sign_in_button, stretch=1)
            user_sign_in_button.clicked.connect(self.user_sign_in)

            user_sign_up_button = QPushButton("Users\nSign-up")
            user_sign_up_button.setStyleSheet("background-color:rgb(32,154,26);font:17px;")
            bar_layout.addWidget(user_sign_up_button, stretch=1)
            user_sign_up_button.clicked.connect(self.user_sign_up)

        products_widget = QWidget()
        products_layout = QGridLayout()
        products_widget.setLayout(products_layout)
        for i, product in enumerate(store.products):
            product_widget = ProductWidget(parent, product, user)
            products_layout.addWidget(product_widget, i//6, i%6)
        scroll = QScrollArea()
        scroll.setWidget(products_widget)

        main_layout = QVBoxLayout()
        main_layout.addLayout(bar_layout)
        main_layout.addWidget(scroll)
        self.setLayout(main_layout)

    def open_cart(self):
        if not self.user:
            error_message = QMessageBox(self)
            error_message.setIcon(QMessageBox.Critical)
            error_message.setWindowTitle("ERROR")
            error_message.setText("You are NOT signed-in!")
            error_message.setStyleSheet("background-color:white")
            error_message.exec()
        else:
            page = CartUi.MainWidget(self.store, self.user)
            self.parent.goto_page(page, [CartUi.MainWidget, self.store, self.user])

    def quit(self):
        self.parent.go_home()

    def seller_sign_in(self):
        page = seller_login(self.parent, self.store)
        self.parent.goto_page(page, [seller_login, self.parent, self.store])

    def seller_sign_up(self):
        page = selelr_signin(self.parent, self.store)
        self.parent.goto_page(page, [selelr_signin, self.parent, self.store])

    def user_sign_in(self):
        page = login(self.parent, self.store)
        self.parent.goto_page(page, [login, self.parent, self.store])

    def user_sign_up(self):
        page = user_signin(self.parent, self.store)
        self.parent.goto_page(page, [user_signin, self.parent, self.store])


"""
RECONSTRUCT multiples files (of pairs) of pulses (at different separations)
AND (optionally) calibrate energies AND (optionally) calculate Energy
resolutions (FWHM)

python recon_resol.py

1) reconstruct energies
2) (optionally) calibrate energies IF input gain scale coefficientes
    are provided
3) (optionally) produce json files with fwhm values pre/post calibration

!!!!!!! Things to review every time it is run (they can change):
        =======================================================
        ** Check resultsDir (nodetSP in case secondaries are not detected,
                           gainScale in case results are for gain scale curves)

        ** SINCE NEWSIMULATIONS (Feb2018): library is always the jitter one
        although applied to non-jitter simulations(bagplots)
"""

# ----IMPORT MODULES --------------
from __future__ import print_function
import os
import math
import shlex
import shutil
# import sys
import tempfile
import json
import sixtevars
from astropy.io import fits
from subprocess import check_call
from reconstruct import reconstruct
from convertEnergies import convertEnergies
import xml.etree.ElementTree as ET

# ----GLOBAL VARIABLES -------------
cwd = os.getcwd()
tmpDir = tempfile.mkdtemp()
print(tmpDir)
# input("stop here")
os.environ["PFILES"] = tmpDir + ":" + os.environ["PFILES"]
os.environ["HEADASNOQUERY"] = ""
os.environ["HEADASPROMPT"] = "/dev/null/"

#
# --------- Read input parameters and PROCESS simulations -----------------
#
if __name__ == "__main__":
    import argparse

    parser = argparse.ArgumentParser(
            description='Reconstruct (and calibrate) (and get ener resol)',
            prog='recon_resol')

    parser.add_argument('--pixName', required=True,
                        help='Extension name in the FITS pixel definition file\
                        (SPA*, LPA1*, LPA2*, LPA3*)')
    parser.add_argument('--labelLib', required=True,
                        help='Label identifying the library (monolib,\
                        multilib, multilibOF, fixedlib, fixedlib2,...)')
    parser.add_argument('--libTmpl', default='LONG',
                        choices=['SHORT', 'LONG'],
                        help='Label identifying the library for the templates:\
                        SHORT or LONG')
    parser.add_argument('--samprate', default="",
                        choices=['', 'samprate2', 'samprate4'],
                        help="baseline, half_baseline, quarter baseline")
    parser.add_argument('--jitter', default="",
                        choices=['', 'jitter', 'jitter_M82_040'],
                        help="no jitter, jitter, M82_040")
    parser.add_argument('--noise', default="",
                        choices=['', 'nonoise'],
                        help="noisy, nonoise")
    parser.add_argument('--decimation', type=int, default=1,
                        help='xifusim decimation factor')
    parser.add_argument('--monoEnergy1', required=True,
                        help='Monochromatic energy (keV) of\
                        input primary simulated pulses')
    parser.add_argument('--monoEnergy2', default="0",
                        help='Monochromatic energy (keV) of\
                        input secondary simulated pulse')
    parser.add_argument('--reconMethod', default='OPTFILT',
                        help='Energy reconstruction Method (OPTFILT,\
                        OPTFILTNMnnnn, WEIGHT, WEIGHTN, I2R, I2RNMnnnn,\
                        I2RALL, I2RNOL, I2RFITTED)')
    parser.add_argument('--filterLength', type=int, default=0,
                        help='filter length for reconstruction')
    parser.add_argument('--nsamples', type=int,
                        help='noise length samples', required=True)
    parser.add_argument('--pulseLength', type=int, required=True,
                        help='pulse length for reconstruction')
    parser.add_argument('--nSimPulses', type=int, required=True,
                        help='pulses in simulations (to locate filenames)')
    parser.add_argument('--nSimPulsesLib', type=int, required=True,
                        help='pulses used to build the library')
    parser.add_argument('--fdomain', default='T',
                        choices=['F', 'T'],
                        help='Optimal Filtering domain (F(req) or T(ime))\
                        [default %(default)s]')
    parser.add_argument('--tstartPulse1', type=int, default=0,
                        help='Start sample for first pulse\
                        [default %(default)s]')
    parser.add_argument('--tstartPulse2', type=int, default=0,
                        help='Start sample for second pulse\
                        [default %(default)s]')
    parser.add_argument('--coeffsFile', default='',
                        help='file with polynomial fit coefficients from\
                        R script polyfit2bias.R')
    parser.add_argument('--detMethod', default='AD',
                        choices=['AD', 'STC'])
    parser.add_argument('--detSP', default=1, type=int,
                        help='are Secondaries to be detected (1:yes; 0:no)')
    parser.add_argument('--resultsDir', default="",
                        help='directory for resulting evt and json files\
                        (from ..../PAIRS/eresol+pixName) tipically\
                        "nodetSP", "detSP" or "gainScale" or ""')
    parser.add_argument('--separations', nargs='+',
                        help='spaced list of separations between\
                        Prim & Sec pulses', default="")
    parser.add_argument('--bbfb', default="",
                        choices=['', 'bbfb', 'bbfb_NewPar', 'bbfb_040',
                                 'bbfb_040_ct', 'fll'], help="dobbfb=n, dobbfb=y")
    parser.add_argument('--Lc', default="",
                        help="Inductance over critical value")
    parser.add_argument('--preBuffer', default=0, type=int,
                        help="preBuffer value for optimal filters")
    parser.add_argument('--Ifit', default=0, type=float,
                        help="Fitted constant for the I2RFITTED expression")
    parser.add_argument('--B0', default=0, type=int,
                        help="B0 (baseline subtraction; B0>0) of \
                        F0 (B0=0) optimal filter")
    parser.add_argument('--LbT', default="0",
                        help="Time (s) to average baseline")
    parser.add_argument('--Sum0Filt', default=0, type=int,
                        help="Optimal Filter SUM shoud be 0? (0=NO; 1=YES)")
    parser.add_argument('--lags', default=1, type=int,
                        help="Do parabola fit if lags=1")
    parser.add_argument('--ct', default="",
                        choices=['', 'ct', 'fit'],
                        help="Filters central part replaced by constant? or\
                        filters derived from largest 8192?")
    inargs = parser.parse_args()

    # print("array=",inargs.array)

    # rename input parameters
    pixName = inargs.pixName
    labelLib = inargs.labelLib
    samprate = inargs.samprate
    jitter = inargs.jitter
    noise = inargs.noise
    bbfb = inargs.bbfb
    Lc = inargs.Lc
    dcmt = inargs.decimation
    mono1EkeV = inargs.monoEnergy1
    mono2EkeV = inargs.monoEnergy2
    reconMethod = inargs.reconMethod
    filterLength = inargs.filterLength
    nsamples = inargs.nsamples
    pulseLength = inargs.pulseLength
    nSimPulses = inargs.nSimPulses
    fdomain = inargs.fdomain
    detMethod = inargs.detMethod
    tstartPulse1 = inargs.tstartPulse1
    tstartPulse2 = inargs.tstartPulse2
    nSimPulsesLib = inargs.nSimPulsesLib
    coeffsFile = inargs.coeffsFile
    libTmpl = inargs.libTmpl
    resultsDir = inargs.resultsDir
    sepsStr = inargs.separations
    detSP = inargs.detSP
    pB = inargs.preBuffer
    Ifit = inargs.Ifit
    LbT = inargs.LbT
    s0 = inargs.Sum0Filt
    lags = inargs.lags
    filterct = inargs.ct
    B0 = inargs.B0

    # general definitions
    EURECAdir = "/dataj6/ceballos/INSTRUMEN/EURECA/"
    ERESOLdir = EURECAdir + "/ERESOL/"
    xifusim = "xifusim" + pixName
    simDir = ERESOLdir + "PAIRS/" + xifusim
    resultsDir = resultsDir.rstrip('\\')
    resultsDir = resultsDir.lstrip('\\')
    if ("gainScale" in resultsDir) or ("base" in resultsDir):
        simDir += "/" + resultsDir
    outDir = ERESOLdir + "PAIRS/eresol" + pixName + "/" + resultsDir
    classAries = ["primaries", "secondaries", "all"]
    if tstartPulse2 == 0:
        classAries = ["all"]

    idxsmp = sixtevars.sampids.index(samprate)
    separation = sixtevars.separations[idxsmp]
    if sepsStr == "":
        sepsStr = [separation]

    # read grading info
    XMLtree = ET.parse(sixtevars.XMLsixte)
    if bbfb == "fll":
        XMLtree = ET.parse(sixtevars.XMLfll)
    XMLroot = XMLtree.getroot()
    for key in XMLroot.findall('grading'):
        num = key.get('num')
        if num == "1":
            invalid = key.get('pre')
            Hres = key.get('post')
            factor = sixtevars.sampfreqs[idxsmp]/sixtevars.sampfreqs[0]
            Hres = int(int(Hres) * factor)
            invalid = int(int(invalid) * factor)

    # to be able to use shorter filters or long filter shortened in FWHM curve
    Hres = min(filterLength, pulseLength)

    print("Hres=", Hres)
    print("filterLength=", filterLength)
    print("pulseLength=", pulseLength)

    # 1) Reconstruct energies
    # ------------------------
    smprtStr, jitterStr, noiseStr, bbfbStr, LcStr, pBStr, IfitStr, LbTStr, \
        s0Str, lagsStr, ctStr, B0str, evtFile, eresolFile = \
        reconstruct(pixName, labelLib, samprate, jitter, dcmt,
                          noise, bbfb, Lc, mono1EkeV, mono2EkeV, reconMethod,
                          filterLength, nsamples, pulseLength, nSimPulses,
                          fdomain, detMethod, tstartPulse1, tstartPulse2,
                          nSimPulsesLib, coeffsFile, libTmpl, simDir, outDir,
                          detSP, pB, Ifit, LbT, s0, lags, filterct, B0, sepsStr)

    # 2) Calibrate (AND/OR) extract Energy resolution info to .json files
    # --------------------------------------------------------------------
    # ---- if calibration parameters are provided:
    #              calculate corrected energies
    #              calculate FWHM, BIAS (calib)
    # ---- if not, calculate only FWMH and BIAS (recons)

    #  define/initialize structure to save all output data

    data = []  # list of dictionaries (one per pulses separation)
    for sep12 in sepsStr:
        dictSep = dict()
        dictBiasErecons = dict()
        dictBiasEreal = dict()
        dictFwhmErecons = dict()
        dictFwhmEreal = dict()
        dictFwhmErealErr = dict()

        print("=============================================")
        print("EXTRACTING FWHMs.........................")
        print("Working in:", outDir)
        print("Setting evtFile: ", evtFile)
        print("Setting eresolFile: ", eresolFile)
        print("=============================================")

        # --------------------------------------------------------------
        # EVENT file processing to calculate FWHM (only for Hres pulses)
        # ---------------------------------------------------------------
        rootEvt = os.path.splitext(evtFile)[0]

        for aries in classAries:  # PRIMARIES / SECONDARIES / ALL
            print("Working with:", aries, "\n")
            if "aries" in aries:  # PRIMARIES // SECONDARIES
                evt = rootEvt + "_" + aries + ".fits"
                if os.path.isfile(evt):
                    os.remove(evt)
            elif aries == "all":
                # evt = evtFile
                evt = rootEvt + "_HR.fits"

            # use only rows with GRADE1==Hres && GRADE2>invalid (Hres pulses)
            try:
                monoEeV = float(mono1EkeV) * 1000.
                if aries == "primaries":
                    comm = "fselect infile=" + evtFile + " outfile=" + evt + \
                           " expr='#ROW%2!=0 && GRADE1==" + str(Hres) +\
                           " && GRADE2>" + str(invalid) + "' clobber=yes"
                elif aries == "secondaries":
                    comm = "fselect infile=" + evtFile + " outfile=" + evt + \
                           " expr='#ROW%2==0 && GRADE1==" + str(Hres) +\
                           " && GRADE2>" + str(invalid) + "' clobber=yes"
                    monoEeV = float(mono2EkeV) * 1000.
                else:
                    # comm = "fselect infile=" + evtFile + " outfile=" + evt +\
                    #       " expr='GRADE1==" + str(Hres) + \
                    #       " && GRADE2>" + str(invalid) + "' clobber=yes"
                    comm = "fselect infile=" + evtFile + " outfile=" + evt +\
                           " expr='GRADE1>=" + str(Hres) + \
                           " && GRADE2>=" + str(Hres) + "' clobber=yes"
                print("Running:", comm)
                args = shlex.split(comm)
                check_call(args, stdout=open(os.devnull, 'wb'))

            except RuntimeError:
                print("Error selecting PRIM/SEC events in evtfile ", evt)
                print(comm)
                os.chdir(cwd)
                shutil.rmtree(tmpDir)
                raise

            # FWHM & BIAS for Erecons
            # read Erecons (SIGNAL) column in numpy array (in keV)
            fpre = fits.open(evt, memmap=True)
            nrows = fpre[1].header["NAXIS2"]
            assert nrows > 0, "Empty evt file (%s): nrows=0 " % evt
            ftab = fpre[1].data
            SIGNALmean = ftab['SIGNAL'].mean()
            SIGNALsigma = ftab['SIGNAL'].std()
            fpre.close()
            del fpre[1].data

            fwhmEreal = 0
            fwhmEreal_err = 0
            biasEreal = 0

            print("evt, SIGNALsigma=", evt, SIGNALsigma)

            fwhm = SIGNALsigma * 2.35 * 1000.
            print("evt, fwhm=", evt, fwhm)

            # FWHM for reconstructed events in eV:
            fwhmErecons = '{0:0.5f}'.format(fwhm)
            # EBIAS = <Erecons> - monoEeV
            bias = SIGNALmean * 1000. - monoEeV
            biasErecons = '{0:0.5f}'.format(bias)
            # print("SIGNALmean=", SIGNALmean)
            # print("monoEeV=", monoEeV)
            # print("biasErecons=", biasErecons, "\n")

            # calculate calibrated energies if polyfit coeffs are provided and
            # previous fwhm is not NaN (some NULL Eners)
            # FWHM only calculated for those pulses reconstructed with HRes
            # if any(a != 0 for a in coeffs) and not math.isnan(fwhm):

            if coeffsFile and not math.isnan(fwhm):
                print("=============================================")
                print("CALIBRATING ENERGIES.........................")
                print("=============================================")

                with open(coeffsFile, "rt") as f:
                    fileCont = f.read()   # JSON file
                    if 'surface' in fileCont:
                        ftype = "surface"
                    elif fileCont[0] == '{':
                        ftype = 'json'
                    else:
                        ftype = 'poly'

                if ftype == 'poly':
                    evtcalib = evt.replace(".fits", ".calib1D")
                    eresolFile = eresolFile.replace(".json", ".json1D")
                elif ftype == 'surface':
                    evtcalib = evt.replace(".fits", ".calib2D")
                    eresolFile = eresolFile.replace(".json", ".json2D")
                elif ftype == 'json':
                    evtcalib = evt.replace(".fits", ".calibItr")
                    eresolFile = eresolFile.replace(".json", ".jsonItr")
                else:
                    raise ValueError("Incorrect Coeffs file type")

                # locate coefficients in coeffs table
                # ------------------------------------
                alias = ("pL" + str(pulseLength) + "_" + detMethod + "_" +
                         fdomain + "_" + labelLib + "_" +
                         reconMethod + str(filterLength) + pBStr + IfitStr + LbTStr +
                         smprtStr + jitterStr + noiseStr + bbfbStr +
                         LcStr + s0Str + lagsStr + ctStr)
                # alias = ("pL" + str(pulseLength) + "_" + detMethod + "_" +
                #         fdomain + "_" + labelLib + "_" +
                #         reconMethod + str(filterLength) + pBStr + IfitStr +
                #         smprtStr + jitterStr + noiseStr + bbfbStr +
                #         LcStr + s0Str + "_base100")
                if "NM" in reconMethod:
                    alias = ("pL" + str(pulseLength) + "_" + detMethod + "_" +
                             fdomain + "_" + labelLib + "_" +
                             reconMethod.split("NM")[0] + str(filterLength) +
                             "NM" + reconMethod.split("NM")[1] + LbTStr +
                             smprtStr + jitterStr + noiseStr + bbfbStr + LcStr)
                # unchecked option...
                # if 'fixedlib' in labelLib and 'OF' not in labelLib:
                #    alias = (detMethod + "_" + labelLib + "OF_" + "_" +
                #             reconMethod + str(pulseLength) + smprtStr +
                #             jitterStr + noiseStr + bbfbStr)
                print("Using alias=", alias)
                convertEnergies(evt, evtcalib, coeffsFile, alias)

                fcal = fits.open(evtcalib, memmap=True)
                nrows = fcal[1].header["NAXIS2"]
                assert nrows > 0, "Empty file (%s): nrows=0 " % evtcalib

                fcaltab = fcal[1].data
                ErealKeVmean = fcaltab['SIGNAL'].mean()
                ErealKeVsigma = fcaltab['SIGNAL'].std()
                fcal.close()
                del fcal[1].data

                print("=============================================")
                print("EXTRACTING corrected FWHMs...................")
                print("=============================================")
                print("Output eresolFile: ", eresolFile, "\n")

                fwhm = ErealKeVsigma * 2.35 * 1000.
                fwhm_err = fwhm / math.sqrt(2 * nrows - 2)
                bias = ErealKeVmean * 1000. - monoEeV
                # FWHM for reconstructed PRIMARIES in eV:
                fwhmEreal = '{0:0.5f}'.format(fwhm)
                fwhmEreal_err = '{0:0.5f}'.format(fwhm_err)
                biasEreal = '{0:0.5f}'.format(bias)
                # print("bias=", bias, "\n")
                # print("...biasEreal=", biasEreal, "\n")

            # write FWHM, BIAS values to dictionary for PRIM/SEC/ALL
            dictFwhmErecons[aries] = fwhmErecons
            dictFwhmEreal[aries] = fwhmEreal
            dictFwhmErealErr[aries] = fwhmEreal_err
            dictBiasErecons[aries] = biasErecons
            dictBiasEreal[aries] = biasEreal
            # END of PRIM & SEC & ALL

        # write data for given separation to a dictionary
        dictSep["separation"] = sep12
        dictSep["fwhmErecons"] = dictFwhmErecons
        dictSep["biasErecons"] = dictBiasErecons
        dictSep["fwhmEreal"] = dictFwhmEreal
        dictSep["biasEreal"] = dictBiasEreal
        dictSep["fwhmEreal_err"] = dictFwhmErealErr

        # save dictionary to list of dictionaries "data"
        data.append(dictSep)

    # Dump total data to JSON file
    feresol = open(eresolFile, 'w')

    json.dump(obj=data, fp=feresol, sort_keys=True)
    feresol.close()
    os.chdir(cwd)
    shutil.rmtree(tmpDir)


from sqlalchemy import create_engine
from sqlalchemy.orm import sessionmaker
from database_setup import Base, Restaurant, MenuItem
engine = create_engine('sqlite:///restaurantmenu.db')
# makes the connection between our class definitions and corresponding tables in
# the database
Base.metadata.bind = engine
# A link of communication between our code executions and the engine created above
DBSession = sessionmaker(bind = engine)
# In order to create, read, update, or delete information on our database,
# SQLAlchemy uses sessions. These are basically just transactions. WE can write
# a bunch of commands and only send them when necessary.
#
# You can call methods from within session (below) to make changes to the
# database. This provides a staging zone for all objects loaded into the
# database session object. Until we call session.commit(), no changes will be
# persisted into the database.
session = DBSession()

myFirstRestaurant = Restaurant(name = "Pizza Plaza")
# staging zone
session.add(myFirstRestaurant)
# add into database
session.commit()

# Check a database to confirm changes
session.query(Restaurant).all()
# This will give hexadecimal location of found objects
cheesepizza = MenuItem(
name = "Cheese Pizza",
description = "Made with all natural goat cheese",
course = "Entree",
price = "$14.99",
restaurant = myFirstRestaurant)

session.add(cheesepizza)
session.commit()
session.query(MenuItem).all()

# Store first result from a query into the Restaurant table
firstResult = session.query(Restaurant).first()
# Access the name column of firstResult
firstResult.name


# Copyright 2016 IBM Corp.
#
# All Rights Reserved.
#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.
import ast

from oslo_log import log as logging

import pypowervm.const as c
from pypowervm.i18n import _
from pypowervm.wrappers import job
from pypowervm.wrappers.virtual_io_server import VIOS

LOG = logging.getLogger(__name__)
_JOB_NAME = "ISCSIDiscovery"
_ISCSI_LOGOUT = "ISCSILogout"

def discover_iscsi(adapter, host_ip, user, password, iqn, vios_uuid):
    """Runs iscsi discovery and login job

    :param adapter: pypowervm adapter
    :param host_ip: The ip address of the iscsi target.
    :param user: The username needed for authentication.
    :param password: The password needed for authentication.
    :param iqn: The IQN (iSCSI Qualified Name) of the created volume on the
                target. (e.g. iqn.2016-06.world.srv:target00)
    :param vios_uuid: The uuid of the VIOS (VIOS must be a Novalink VIOS type).
    :return: The device name of the created volume.
    :return: The UniqueDeviceId of the create volume.
    """

    resp = adapter.read(VIOS.schema_type, vios_uuid,
                        suffix_type=c.SUFFIX_TYPE_DO, suffix_parm=(_JOB_NAME))
    job_wrapper = job.Job.wrap(resp)

    # Create job parameters
    job_parms = [job_wrapper.create_job_parameter('hostIP', host_ip)]
    job_parms.append(job_wrapper.create_job_parameter('password', password))
    job_parms.append(job_wrapper.create_job_parameter('user', user))
    job_parms.append(job_wrapper.create_job_parameter('targetIQN', iqn))

    job_wrapper.run_job(vios_uuid, job_parms=job_parms, timeout=120)
    results = job_wrapper.get_job_results_as_dict()

    # DEV_OUTPUT: ["IQN1 dev1 udid", "IQN2 dev2 udid"]
    output = ast.literal_eval(results.get('DEV_OUTPUT'))
    # Find dev corresponding to given IQN
    for dev in output:
        if len(dev.split()) != 3:
            LOG.warning(_("Invalid device output: %(dev)s"), {'dev': dev})
            continue
        outiqn, outname, udid = dev.split()
        if outiqn == iqn:
            return outname, udid
    LOG.error(_("Expected IQN: %(IQN)s not found on iscsi target %(host_ip)s"),
              {'IQN': iqn, 'host_ip': host_ip})
    return None, None

def discover_iscsi_initiator(adapter, vios_uuid):
    """Discovers the initiator name.

    :param adapter: pypowervm adapter
    :param vios_uuid: The uuid of the VIOS (VIOS must be a Novalink VIOS type).
    :return: The iscsi initiator name.
    """
    resp = adapter.read(VIOS.schema_type, vios_uuid,
                        suffix_type=c.SUFFIX_TYPE_DO, suffix_parm=(_JOB_NAME))
    job_wrapper = job.Job.wrap(resp)

    job_wrapper.run_job(vios_uuid, timeout=120)
    results = job_wrapper.get_job_results_as_dict()

    # InitiatorName: iqn.2010-10.org.openstack:volume-4a75e9f7-dfa3
    return results.get('InitiatorName')

def remove_iscsi(adapter, targetIQN, vios_uuid):
    """Logout of an iSCSI session.

    The iSCSI volume with the given targetIQN must not have any mappings from
    the VIOS to a client when this is called.

    :param adapter: pypowervm adapter
    :param targetIQN: The IQN (iSCSI Qualified Name) of the created volume on
                      the target. (e.g. iqn.2016-06.world.srv:target00)
    :param vios_uuid: The uuid of the VIOS (VIOS must be a Novalink VIOS type).
    """
    resp = adapter.read(VIOS.schema_type, vios_uuid,
                        suffix_type=c.SUFFIX_TYPE_DO,
                        suffix_parm=(_ISCSI_LOGOUT))
    job_wrapper = job.Job.wrap(resp)

    job_parms = [job_wrapper.create_job_parameter('targetIQN', targetIQN)]
    job_wrapper.run_job(vios_uuid, job_parms=job_parms, timeout=120)


from github import Github, InputGitAuthor, GithubException
import json
import os
from retry import retry
import sys
import time
import urllib.request

HTTP_REQUEST_RETRIES = 3
HTTP_REQUEST_DELAY_IN_SECONDS = 2
HTTP_REQUEST_DELAY_MULTIPLIER = 2

ORGANIZATION = "ballerina-platform"
LANG_VERSION_KEY = "ballerinaLangVersion"
VERSION_KEY = "version="

LANG_VERSION_UPDATE_BRANCH = 'automated/stdlib_version_update'
MASTER_BRANCH = "master"
MAIN_BRANCH = "main"

packageUser = os.environ["packageUser"]
packagePAT = os.environ["packagePAT"]
packageEmail = os.environ["packageEmail"]

ENCODING = "utf-8"

OPEN = "open"
MODULES = "modules"

COMMIT_MESSAGE_PREFIX = "[Automated] Update lang version to "
PULL_REQUEST_BODY_PREFIX = "Update ballerina lang version to `"
PULL_REQUEST_TITLE = "[Automated] Update Dependencies"

MODULE_LIST_FILE = "release/resources/module_list.json"
BALLERINA_DISTRIBUTION = "ballerina-distribution"
PROPERTIES_FILE = "gradle.properties"

def main():
    lang_version = get_lang_version()
    module_list_json = get_module_list_json()
    check_and_update_lang_version(module_list_json, lang_version)

def get_lang_version():
    try:
        properties = open_url(
            "https://raw.githubusercontent.com/ballerina-platform/ballerina-lang/master/gradle.properties")
    except Exception as e:
        print('Failed to gradle.properties file in ballerina-lang' + e)
        sys.exit(1)

    for line in properties:
        line = line.decode(ENCODING).strip()
        if line.startswith(VERSION_KEY):
            return line.split("=")[-1]

@retry(
    urllib.error.URLError,
    tries=HTTP_REQUEST_RETRIES,
    delay=HTTP_REQUEST_DELAY_IN_SECONDS,
    backoff=HTTP_REQUEST_DELAY_MULTIPLIER
)
def open_url(url):
    return urllib.request.urlopen(url)

def get_module_list_json():
    try:
        with open(MODULE_LIST_FILE) as f:
            module_list = json.load(f)

    except Exception as e:
        print(e)
        sys.exit(1)

    # Append ballerina distribution to the list to update the lang version
    module_list[MODULES].append(BALLERINA_DISTRIBUTION)
    return module_list

def check_and_update_lang_version(module_list_json, lang_version):
    for module_name in module_list_json[MODULES]:
        update_module(module_name, lang_version)

def update_module(module_name, lang_version):
    github = Github(packagePAT)
    repo = github.get_repo(ORGANIZATION + "/" + module_name)
    try:
        properties_file = repo.get_contents(PROPERTIES_FILE, ref=LANG_VERSION_UPDATE_BRANCH)
    except:
        properties_file = repo.get_contents(PROPERTIES_FILE)

    properties_file = properties_file.decoded_content.decode(ENCODING)
    update, updated_properties_file = get_updated_properties_file(module_name, properties_file, lang_version)
    if update:
        commit_changes(repo, updated_properties_file, lang_version)
        create_pull_request(repo, lang_version)
        time.sleep(30)

def get_updated_properties_file(module_name, properties_file, lang_version):
    updated_properties_file = ""
    update = False

    for line in properties_file.splitlines():
        updated_properties_file += line + "\n"

    return update, updated_properties_file

def commit_changes(repo, updated_file, lang_version):
    author = InputGitAuthor(packageUser, packageEmail)
    try:
        base = repo.get_branch(MASTER_BRANCH)
    except:
        base = repo.get_branch(MAIN_BRANCH)

    try:
        ref = f"refs/heads/" + LANG_VERSION_UPDATE_BRANCH
        repo.create_git_ref(ref=ref, sha=base.commit.sha)
    except :
        try:
            repo.get_branch(LANG_VERSION_UPDATE_BRANCH)
            repo.merge(LANG_VERSION_UPDATE_BRANCH, base.commit.sha, "Sync default branch")
        except GithubException as e:
            print("Error occurred: " + e)

    current_file = repo.get_contents(PROPERTIES_FILE, ref=LANG_VERSION_UPDATE_BRANCH)
    repo.update_file(
        current_file.path,
        COMMIT_MESSAGE_PREFIX + lang_version,
        updated_file,
        current_file.sha,
        branch=LANG_VERSION_UPDATE_BRANCH,
        author=author
    )

def create_pull_request(repo, lang_version):
    pulls = repo.get_pulls(state=OPEN, head=LANG_VERSION_UPDATE_BRANCH)
    pr_exists = False

    for pull in pulls:
        if PULL_REQUEST_TITLE in pull.title:
            pr_exists = True

    if not pr_exists:
        try:
            repo.create_pull(
                title=PULL_REQUEST_TITLE,
                body=PULL_REQUEST_BODY_PREFIX + lang_version + "`",
                head=LANG_VERSION_UPDATE_BRANCH,
                base=MASTER_BRANCH
            )
        except:
            repo.create_pull(
                title=PULL_REQUEST_TITLE,
                body=PULL_REQUEST_BODY_PREFIX + lang_version + "`",
                head=LANG_VERSION_UPDATE_BRANCH,
                base=MAIN_BRANCH
            )

main()


def create_ego_adj_list(fp):
    ego_adj_dict = { }

    for line in fp:
        # read and tokenize line, convert to integers for sorting
        line = [ int(elem) for elem in line.split() ]
        # ensure ego, if present, is in position 0 by sorting in place
        line.sort()
        # ego
        if line[0] <= 10000:
            # add key if not present
            if not line[0] in ego_adj_dict:
                ego_adj_dict[line[0]] = [ ]
            # add non-ego nodes
            for i in xrange(1, len(line)):
                ego_adj_dict[line[0]].append(line[i])

    return ego_adj_dict

# file to process
FILES=[ '/opt2/D4D/data/SET4/raw/tsv/GRAPHS_'+str(i)+'.TSV' \
    for i in xrange(0,10) ]

for i in xrange(0,len(FILES)):
    # create a file pointer, open data file for reading
    fp_in = open(FILES[i], 'r')

    # generate dictionary of egos and connected nodes
    ego_adj_dict = create_ego_adj_list(fp_in)

    # create a file pointer for ego adjacency lists
    fp_out = open('set4_t'+str(i)+'_ego_adj_list.txt', 'w')

    # 1) create a list of all keys in ego dictionary and sort
    # 2) create strings of each ego adjacency list, creating lists
    #    and sorting along the way as necessary
    # 3) write strings to file
    ego_list = ego_adj_dict.keys()
    ego_list.sort()

    for ego in ego_list:
        s = str(ego)
        non_ego_list = ego_adj_dict[ego]
        non_ego_list.sort()
        for non_ego in non_ego_list:
            s = s + ' ' + str(non_ego)
        s = s + '\n'
        fp_out.write(s)

    # close file pointers
    fp_out.close()
    fp_in.close()


import user
import operator

from django.contrib import messages
from django.db.models import Q
from django.shortcuts import render, get_object_or_404, redirect
from django.contrib.auth.decorators import login_required
from django.views.generic import TemplateView, DetailView, ListView

from .forms import NewQueryForm, NewAnswerForm, EditQueryForm, CloseQueryForm, UserRegistrationForm, UserEditForm, \
  ProfileEditForm, CommentForm, AnswereditForm, CorrectAnswerForm

from .models import Post, Thread, Profile, Comment, User, Correctanswer

def user_registration(request):
  if request.method == 'POST':
    user_form = UserRegistrationForm(request.POST)
    if user_form.is_valid():
      new_user = user_form.save(commit=False)
      new_user.set_password(user_form.cleaned_data['password'])
      new_user.save()
      profile = Profile.objects.create(user=new_user)
      return render(request, 'forum/register_done.html', {'new_user': new_user})
  else:
    user_form = UserRegistrationForm()
  return render(request, 'forum/signup.html', {'user_form': user_form})

@login_required
def user_edit(request):
  if request.method == 'POST':
    user_form = UserEditForm(instance=request.user, data=request.POST)
    profile_form = ProfileEditForm(instance=request.user.profile,
                                   data=request.POST,
                                   files=request.FILES)
    if user_form.is_valid() and profile_form.is_valid():
      user = user_form.save()
      profile = profile_form.save()
      messages.success(request, 'Profile Updated successfully')
    else:
      messages.error(request, 'Error updating your profile')

  else:
    user_form = UserEditForm(instance=request.user)
    profile_form = ProfileEditForm(instance=request.user.profile)
  return render(request, 'forum/profile_edit.html', {'user_form': user_form, 'profile_form': profile_form })

@login_required
def newquery(request):
  profile = Profile.objects.all()
  userquestions = Post.objects.filter(user__username=request.user)
  useranswers = Thread.objects.filter(user__username=request.user)
  if request.method == 'POST':
    query_form = NewQueryForm(request.POST)
    if query_form.is_valid():
      new_query = query_form.save(commit=False)
      new_query.user = request.user
      new_query.save()
      messages.success(request, "Query added Succesfully")
      return render(request, 'forum/app/userquerylist.html')
    else:
      messages.error(request, "Query not added ")
  else:
    query_form = NewQueryForm()
  return render(request, "forum/app/newquery.html", {"query_form": query_form,
                                                     "profile": profile,
                                                     "userquestions":userquestions,
                                                     "useranswers":useranswers})

@login_required
def user_query(request):
  post = Post.objects.filter(user__username=request.user)
  return render(request, "forum/app/userquerylist.html", {'post': post})

@login_required
def user_answer(request):
  answers = Thread.objects.filter(user__username=request.user)
  return render(request, 'forum/app/useranswerlist.html', {'answers': answers})

@login_required
def userquery_edit(request, slug):
  edit = get_object_or_404(Post, slug=slug)
  if request.user == edit.user and edit.close == False:
    if request.method == 'POST':
      # instance collect the current question data and update it in save method
      editform = EditQueryForm(instance=edit, data=request.POST)
      if editform.is_valid():
        update = editform.save(commit=False)
        update.user = request.user
        update.save()
        messages.success(request, "Answer update successfully")
      else:
        messages.error(request, "Answer not updated")
    else:
      editform = EditQueryForm(instance=edit)
    return render(request, 'forum/app/userqueryedit.html', {'edit': edit,
                                                            'editform': editform,})
  else:
    return render(request, 404)

def useranswer_edit(request, id, slug):
  query = get_object_or_404(Post, slug=slug)
  answers = query.answer.filter(id=id)
  # foreign key field
  question = Post.objects.get(title=query.title)
  answer = Thread.objects.get(id=id)
  if request.user == answer.user and query.close == False:
    if request.method == 'POST':
      editform = AnswereditForm(instance=answer, data=request.POST)
      if editform.is_valid():
        update = editform.save(commit=False)
        update.user = request.user
        update.post = question
        update.save()
        messages.success(request, "Answer update successfully")
      else:
        messages.error(request, "Answer not updated")
    else:
      editform = AnswereditForm(instance=answer)
    return render(request, 'forum/app/useransweredit.html', {'editform': editform,
                                                             'query': query,
                                                             'answers': answers,
                                                             })

def userquery_close(request, slug):
  edit = get_object_or_404(Post, slug=slug)
  if request.user == edit.user and edit.close == False:
    if request.method == 'POST':
      closeform = CloseQueryForm(instance=edit, data=request.POST)
      if closeform.is_valid():
        closeform.save()
        messages.success(request, 'question closed successfully')
      else:
        messages.error(request, 'question unable to close')
    else:
      closeform = CloseQueryForm(instance=edit)
    return render(request, 'forum/app/userqueryclose.html', {'closeform': closeform})
  else:
    return render(request, 404)

def querydetail(request, year, month, day, slug):
  query = get_object_or_404(Post, slug=slug,
                            create__year=year,
                            create__month=month,
                            create__day=day)
  # query details
  answers = query.answer.all()
  comment = Comment.objects.filter(answer__in=answers)

  # query new answer foreign key
  question = Post.objects.get(title=query.title)
  answereach = question.answer.filter()

  correct = query.correctanswer_post.filter()

  if 'post_answer' in request.POST:
    answerform = NewAnswerForm(data=request.POST)
    if answerform.is_valid():
      newanswer = answerform.save(commit=False)
      newanswer.user = request.user
      # verify user anwser exist
      if answereach.filter(user__username=newanswer.user).exists():
        messages.error(request, "answer exist")
      else:
        newanswer.post = question
        newanswer.save()
        messages.success(request, "Newanswer added sucessfully")
    else:
      messages.error(request, "Newanswer not added")

  if 'correct_answer' in request.POST:
    correctanswer = CorrectAnswerForm(data=request.POST)
    if correctanswer.is_valid():
      custom_correctanswer = correctanswer.save(commit=False)
      custom_correctanswer.user = request.user
      custom_correctanswer.post = question
      answer_id = request.POST.get('correctanswer_id', "")
      answer = Thread.objects.get(id=answer_id)
      custom_correctanswer.answer = answer
      if correct.count() < 1:
        custom_correctanswer.save()
        messages.success(request, "correct answer added successfully")
      else:
        messages.error(request, "Correct answer added already. You can't change")
    else:
      messages.error(request, "correct Answer not added")

  if'post_comment' and request.POST:
    commentform = CommentForm(request.POST)
    if commentform.is_valid():
      new_comment = commentform.save(commit=False)
      answer_id = request.POST.get('answer_id', "")
      answer = Thread.objects.get(id=answer_id)
      obj, created = Comment.objects.get_or_create(user=request.user, \
                                                   post=question, \
                                                   answer=answer, comment=new_comment)
      messages.success(request, 'comment added sucessfully')

  answerform = NewAnswerForm()
  correctanswer = CorrectAnswerForm()
  commentform = CommentForm()
  return render(request, 'forum/app/querydetail.html', {'query': query,
                                                        'answers': answers,
                                                        'comment': comment,
                                                        'answerform': answerform,
                                                        'correct': correct,
                                                        'correctanswer': correctanswer,
                                                        'commentform': commentform})

def all_question(request):
  list = Post.objects.filter()

  # search all the queries
  search_word = request.GET.get("query")
  if search_word:
    # list = list.annotate(search=SearchVector('title','description')).filter(search=search_word)
    # Stemmer algorithm not used
    list = list.filter(
      Q(title__icontains=search_word) |
      Q(description__icontains=search_word)
    ).distinct()
  return render(request, "forum/app/allquestion.html", {'list': list})

class RecentView(ListView):
  model = Post
  queryset = Post.objects.all().order_by('-create')
  context_object_name = 'recent'
  template_name = 'forum/app/recent.html'

class HomeView(ListView):
  model = Post
  paginate_by = '30'
  queryset = Post.objects.all().order_by('-create')
  context_object_name = "recent"
  template_name = 'forum/app/index.html'

  def get_queryset(self):
    result = super(HomeView, self).get_queryset()
    post = Post.objects.all()
    query = self.request.GET.get('query')
    if query:
      query_list = query.split()
      result = post.filter(
        reduce(operator.and_,
               (Q(title__icontains=q) for q in query_list)) |
        reduce(operator.and_,
               (Q(description__icontains=q) for q in query_list))
      )

    return result

  def get_context_data(self, **kwargs):
    context = super(HomeView, self).get_context_data(**kwargs)
    context['announcements'] = Post.objects.all().filter(category=Post.ANNOUNCEMENT)
    context['general'] = Post.objects.all().filter(category=Post.GENERAL)
    context['bugreport'] = Post.objects.all().filter(category=Post.BUGREPORT)
    context['tips'] = Post.objects.all().filter(category=Post.TIPSANDTRICKS)
    return context

class AnnouncementView(ListView):
  model = Post
  paginate_by = '10'
  queryset = Post.objects.all().filter(category=Post.ANNOUNCEMENT)
  context_object_name = "announcement"
  template_name = 'forum/app/announcement.html'

class GeneralView(ListView):
  model = Post
  paginate_by = '10'
  context_object_name = "general"
  queryset = Post.objects.all().filter(category=Post.GENERAL)
  template_name = 'forum/app/general.html'

class BugreportView(ListView):
  model = Post
  paginate_by = '10'
  context_object_name = "bugreport"
  queryset = Post.objects.all().filter(category=Post.BUGREPORT)
  template_name = 'forum/app/bugreport.html'

class TipsView(ListView):
  model = Post
  paginate_by = '10'
  context_object_name = "tips"
  queryset = Post.objects.all().filter(category=Post.TIPSANDTRICKS)
  template_name = 'forum/app/tips.html'

class UserView(DetailView):
  model = User
  context_object_name = "user"
  template_name = 'forum/app/userdetail.html'

  def get_context_data(self, **kwargs):
    context = super(UserView, self).get_context_data(**kwargs)
    context['profile'] = Profile.objects.all().get(user=self.request.user)
    context['userquestions'] = Post.objects.filter(user__username=self.request.user)
    context['useranswers'] = Thread.objects.filter(user__username=self.request.user)
    return context


# Copyright 2015 OpenStack Foundation
# All Rights Reserved.
#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.

import datetime

from oslotest import base as test_base

from oslo_utils import fixture
from oslo_utils.fixture import keystoneidsentinel as keystids
from oslo_utils.fixture import uuidsentinel as uuids
from oslo_utils import timeutils
from oslo_utils import uuidutils

class TimeFixtureTest(test_base.BaseTestCase):

    def test_set_time_override_using_default(self):
        # When the fixture is used with its default constructor, the
        # override_time is set to the current timestamp.
        # Also, when the fixture is cleaned up, the override_time is reset.

        self.assertIsNone(timeutils.utcnow.override_time)
        with fixture.TimeFixture():
            self.assertIsNotNone(timeutils.utcnow.override_time)
        self.assertIsNone(timeutils.utcnow.override_time)

    def test_set_time_override(self):
        # When the fixture is used to set a time, utcnow returns that time.

        new_time = datetime.datetime(2015, 1, 2, 3, 4, 6, 7)
        self.useFixture(fixture.TimeFixture(new_time))
        self.assertEqual(new_time, timeutils.utcnow())
        # Call again to make sure it keeps returning the same time.
        self.assertEqual(new_time, timeutils.utcnow())

    def test_advance_time_delta(self):
        # advance_time_delta() advances the overridden time by some timedelta.

        new_time = datetime.datetime(2015, 1, 2, 3, 4, 6, 7)
        time_fixture = self.useFixture(fixture.TimeFixture(new_time))
        time_fixture.advance_time_delta(datetime.timedelta(seconds=1))
        expected_time = datetime.datetime(2015, 1, 2, 3, 4, 7, 7)
        self.assertEqual(expected_time, timeutils.utcnow())

    def test_advance_time_seconds(self):
        # advance_time_seconds() advances the overridden time by some number of
        # seconds.

        new_time = datetime.datetime(2015, 1, 2, 3, 4, 6, 7)
        time_fixture = self.useFixture(fixture.TimeFixture(new_time))
        time_fixture.advance_time_seconds(2)
        expected_time = datetime.datetime(2015, 1, 2, 3, 4, 8, 7)
        self.assertEqual(expected_time, timeutils.utcnow())

class UUIDSentinelsTest(test_base.BaseTestCase):

    def test_different_sentinel(self):
        uuid1 = uuids.foobar
        uuid2 = uuids.barfoo
        self.assertNotEqual(uuid1, uuid2)
        keystid1 = keystids.foobar
        keystid2 = keystids.barfoo
        self.assertNotEqual(keystid1, keystid2)

    def test_returns_uuid(self):
        self.assertTrue(uuidutils.is_uuid_like(uuids.foo))
        self.assertTrue(uuidutils.is_uuid_like(keystids.foo))

    def test_returns_string(self):
        self.assertIsInstance(uuids.foo, str)
        self.assertIsInstance(keystids.foo, str)

    def test_with_underline_prefix(self):
        ex = self.assertRaises(AttributeError, getattr, uuids, '_foo')
        self.assertIn("Sentinels must not start with _", str(ex))
        ex = self.assertRaises(AttributeError, getattr, keystids, '_foo')
        self.assertIn("Sentinels must not start with _", str(ex))


import cvxpy as cvx
import numpy as np

def cvxSolElasticNet(n_cols,b,A,lam1,lam2):
    x_cvx = cvx.Variable(n_cols)
    bmat = np.matrix(b)
    bmat = bmat.T
    Amat = np.matrix(A)
    f = cvx.sum_squares(Amat * x_cvx - bmat) / 2 + lam1 * cvx.norm(x_cvx, 1) + lam2 * 0.5 * cvx.norm(x_cvx,2)**2
    problem = cvx.Problem(cvx.Minimize(f))
    problem.solve()
    opt = problem.value
    xopt = x_cvx.value
    return [opt,xopt]

def cvxSolL1LS(n_cols,b,A,lam,boxConstraint,boxMx,boxMin,l2Constraint,tauL2):
    x_cvx = cvx.Variable(n_cols)
    bmat = np.matrix(b)
    bmat = bmat.T
    Amat = np.matrix(A)
    f = cvx.sum_squares(Amat * x_cvx - bmat) / 2 + lam * cvx.norm(x_cvx, 1)

    aConstraint = False
    if (boxConstraint):
        aConstraint = True
        if(l2Constraint):
            constraints = [x_cvx >= boxMin, x_cvx <= boxMx, cvx.norm(x_cvx,2)<=tauL2]
        constraints = [x_cvx >= boxMin, x_cvx <= boxMx]
    elif(l2Constraint):
        constraints = [cvx.norm(x_cvx, 2) <= tauL2]
        aConstraint = True

    if(aConstraint):
        problem = cvx.Problem(cvx.Minimize(f), constraints)
    else:
        problem = cvx.Problem(cvx.Minimize(f))

    problem.solve()
    opt = problem.value
    xopt = x_cvx.value
    NNZ = sum([abs(xopt[i])>1e-6 for i in range(xopt.shape[0])])
    print('Number nonzero in solution is ' + str(float(NNZ)) + ' out of '+ str(n_cols) )
    return [opt,xopt]

def cvxSolLR(n_cols,b,A,lam,tauL2):
    x_cvx = cvx.Variable(n_cols)
    Amat = np.matrix(A)
    constraint = [cvx.norm(x_cvx,2)<=tauL2]
    f = cvx.sum_entries(cvx.logistic(-cvx.mul_elemwise(b,Amat * x_cvx)))+ lam * cvx.norm(x_cvx, 1)
    if(tauL2<float('inf')):
        problem = cvx.Problem(cvx.Minimize(f),constraint)
    else:
        problem = cvx.Problem(cvx.Minimize(f))
    problem.solve()
    opt = problem.value
    xopt = x_cvx.value
    return [opt,xopt]

def cvxSolSVM(n_cols,b,A,lam):
    x_cvx = cvx.Variable(n_cols)
    bmat = np.matrix(b)
    Amat = np.matrix(A)
    f = cvx.sum_entries(cvx.pos(1-cvx.mul_elemwise(b,Amat*x_cvx))) + 0.5*lam*cvx.norm(x_cvx,2)**2

    problem = cvx.Problem(cvx.Minimize(f))

    problem.solve()
    opt = problem.value
    xopt = x_cvx.value
    return [opt,xopt]

def cvxSolvePortfolio(Q,m,r):
    [d,_] = Q.shape
    x = cvx.Variable(d)
    f = 0.5*cvx.quad_form(x,Q)
    constraints = [cvx.sum_entries(x) == 1.0, x>=0, cvx.sum_entries(cvx.mul_elemwise(m,x)) >= r]
    problem = cvx.Problem(cvx.Minimize(f),constraints)

    problem.solve(solver='CVXOPT')
    opt = problem.value
    xopt = np.array(x.value).reshape(d)
    return [opt,xopt]


import logging
from datetime import datetime

def _map_from_record(result, applications):
    res = {}
    _source = result['_source']
    try:
        res['timestamp'] = _source['@timestamp']
        res['message'] = _source['message']
        res['host'] = _source['host']
        if ('_type' in result) and (result['_type'] == "supervisor"):
            res['class_name'] = ''
            res['level'] = ''
            res['thread_name'] = ''
        elif ('_type' in result) and (result['_type'] == "python"):
            res['class_name'] = _source['filename']
            res['level'] = _source['levelname']
            res['thread_name'] = ''

            # did the dev's give us a stacktrace?
            if 'exception' in _source:
                res['exception'] = u"\n{0}".format("".join(_source['exception']))
        else:
            res['class_name'] = _source['class']
            res['level'] = _source['level']
            if 'thread_name' in _source:
                res['thread_name'] = "[{0}]".format(_source['thread_name'])
            else:
                res['thread_name'] = ''

            # did the dev's give us a stacktrace?
            if 'exception' in _source:
                res['exception'] = u"\n{0} {1}".format(_source['exception']['exception_class'], _source['exception']['stacktrace'])
    except KeyError as ke:
        if ke:
            logging.info(
                "{0} Bad Log for application(s) {1} on field {2}\n".format(datetime.now(), applications, ke[0]))
        return None

    return res

class EstailLogparse():
    def fields(self):
        return (
            "@timestamp",
            "exception",
            "message",
            "class",
            "filename",
            "level",
            "levelname",
            "host",
            "thread_name",
        )

    def to_message(self, result):
        res_map = _map_from_record(result, args.applications)
        if res_map is None:
            return None

        message = u"{0} {1} {2} [{3}] {4} {5}".format(
            res_map['timestamp'], res_map['host'], res_map['level'], res_map['class_name'], res_map['thread_name'], res_map['message']).encode('utf-8')
        if 'exception' in res_map:
            # better hand this exception stuff over
            message += res_map['exception'].encode('utf-8')
        return message


# Tencent is pleased to support the open source community by making TNN available.
#
# Copyright (C) 2020 THL A29 Limited, a Tencent company. All rights reserved.
#
# Licensed under the BSD 3-Clause License (the "License"); you may not use this file except
# in compliance with the License. You may obtain a copy of the License at
#
# https://opensource.org/licenses/BSD-3-Clause
#
# Unless required by applicable law or agreed to in writing, software distributed
# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
# CONDITIONS OF ANY KIND, either express or implied. See the License for the
# specific language governing permissions and limitations under the License.

from utils import cmd
from utils import checker
from utils import return_code
from . import align_model

from converter import logging

import os
import sys

def throw_exception(current_shape):
    message = "Current shape: "
    for item in current_shape:
        name, shape = item
        message += str(name) + ": " + str(shape) + "   "

    logging.error("You should use -in to specify input's name and shape. e.g.: -in name[1,3,32,32]")
    logging.error(message)

    sys.exit(return_code.CONVERT_FAILED)

def convert(onnx_path, output_dir=None, version="v1.0", optimize=True, half=False, align=False,
            input_path=None, refer_path=None, input_names : str=None):
    """
    执行 onnx 转换为 tnn 的转换指令
    :parameter:
          onnx_path:    需要转换的 onnx 文件的路径
          output_path:  生成的 tnn 文件的路径
          version:      转换模型的版本号
          optimize:     是否需要对模型进行优化,默认是需要进行优化
          half:         是否需要转为 FP16 的模型,减小模型的大小
    :return return_code
    :exception 执行超时
    """
    logging.info("Converter ONNX to TNN Model\n")

    checker.check_file_exist(onnx_path)

    ret, current_shape = checker.check_onnx_dim(onnx_path)

    if ret is False and current_shape is not None:
        if input_names is None:
            throw_exception(current_shape)
        if input_names is not None and not ("[" in input_names and "]" in input_names):
            throw_exception(current_shape)

    proto_suffix = '.tnnproto'
    model_suffix = '.tnnmodel'
    command = "python3 onnx2tnn.py " + onnx_path
    command = command + " -version=v1.0"
    checker.check_file_exist(onnx_path)
    if optimize is True:
        command = command + " -optimize=1"
    else:
        command = command + " -optimize=0"
    if half is True:
        command = command + " -half=1"
    else:
        command = command + " -half=0"

    if output_dir is None:
        output_dir = os.path.dirname(onnx_path)
    checker.check_file_exist(output_dir)
    command = command + " -o " + output_dir

    if input_names is not None:
        new_input_names = ""
        for char in input_names:
            if char == "[":
                char = ":"
            if char == "]":
                continue
            new_input_names += char
        command = command + " -input_shape " + new_input_names
    logging.debug("The onnx2tnn command:" + command + "\n")

    work_dir = "../onnx2tnn/onnx-converter/"
    result = cmd.run(command, work_dir=work_dir)

    if result == 0:
        logging.info("Converter ONNX to TNN model succeed!\n")
    else:
        logging.error("Converter ONNX to TNN model failed!\n")
        sys.exit(return_code.CONVERT_FAILED)
    onnx_base_name = os.path.basename(onnx_path)

    if align is True:
        if optimize is True:
            tnn_proto_name = onnx_base_name[:-len('.onnx')] + '.opt' + proto_suffix
            tnn_model_name = onnx_base_name[:-len('.onnx')] + '.opt' + model_suffix
        else:
            tnn_proto_name = onnx_base_name[:-len('.onnx')] + proto_suffix
            tnn_model_name = onnx_base_name[:-len('.onnx')] + model_suffix
        tnn_proto_path = os.path.join(output_dir, tnn_proto_name)
        tnn_model_path = os.path.join(output_dir, tnn_model_name)

        if input_names is None:
            align_model.align_model(onnx_path, tnn_proto_path, tnn_model_path, input_path, refer_path)
        else:
            align_model.align_model(onnx_path, tnn_proto_path, tnn_model_path, input_path, refer_path, new_input_names)


from typing import Any

kernel_src: str
dev: Any
context: Any
queue: Any
prog: Any
v: Any
v_buff: Any
user_event: Any

def read_complete(status: Any, data: Any) -> None: ...

global_size: Any
local_size: Any
kernel_event: Any
read_event: Any


from django.contrib.auth.models import User
from django.utils.translation import ugettext_lazy as _, ugettext
from django.shortcuts import get_object_or_404

from swapleaf.app.friends.models import Friendship, FriendshipRequest
from swapleaf.helper.common import get_user_login_object, convert_queryset_to_list, remove_duplicate_object
from swapleaf.helper.member import check_partnership
from swapleaf.app.main.models import BookTransaction

from allauth.account import app_settings
from allauth.utils import email_address_exists

def get_partner_match_number(user1,user2):
	list_buy_book1 = user1.get_profile().buy_book.all()
	list_buy_book2 = user2.get_profile().buy_book.all()
	list_transaction_book1 = BookTransaction.objects.filter(seller=user1)
	list_transaction_book2 = BookTransaction.objects.filter(seller=user2)
	matches = 0
	for book_transaction in list_transaction_book1:
		for book_buying in list_buy_book2:
				if book_transaction.book == book_buying.book:
					matches += 1

	for book_transaction in list_transaction_book2:
		for book_buying in list_buy_book1:
				if book_transaction.book == book_buying.book:
					matches += 1
	return matches

def get_partner_match_value(user1,user2):
	list_buy_book1 = user1.get_profile().buy_book.all()
	list_buy_book2 = user2.get_profile().buy_book.all()
	list_transaction_book1 = BookTransaction.objects.filter(seller=user1)
	list_transaction_book2 = BookTransaction.objects.filter(seller=user2)
	result = []
	for book_transaction in list_transaction_book1:
		for book_buying in list_buy_book2:
				if book_transaction.book == book_buying.book:
					result.append((book_transaction,user1,user2))

	for book_transaction in list_transaction_book2:
		for book_buying in list_buy_book1:
				if book_transaction.book == book_buying.book:
					result.append((book_transaction,user2,user1))
	return result

def get_partners_data(request):
	user_login = get_user_login_object(request)
	list_buy_book = convert_queryset_to_list(user_login.get_profile().buy_book.all())
	list_transaction_book = BookTransaction.objects.filter(seller=user_login)
	result = []
	#users = User.objects.all()
	partners = user_login.get_profile().partners.all()
	for user in partners:
		#if check_partnership(request,user.username) == 1 and user != user_login:
		book_transactions1 = BookTransaction.objects.filter(seller=user)
		matches = 0
		partner_match = 0

		# Get all the book the user_login want to buy and the partner want to sell/trade
		for book_transaction in book_transactions1:
			for book_buying in list_buy_book:
				if book_transaction.book == book_buying.book:
					matches += 1

		# Get all the book the user_login want to sell/trade and the partner want to buy
		for book_transaction in list_transaction_book:
			for book_buying in user.get_profile().buy_book.all():
				if book_transaction.book == book_buying.book:
					matches += 1

		for user2 in partners:
			if user2 != user:
				partner_match += get_partner_match_number(user2,user)

		# Get the match for partners of the user_login's partners
		# for partner in user.get_profile().partners.all():
		# 	if partner != user_login:
		# 		book_transactions2 = BookTransaction.objects.filter(seller=partner)
		# 		m1 = 0
		# 		for book_transaction in book_transactions2:
		# 			for book_buying in list_buy_book:
		# 				if book_transaction.book == book_buying.book:
		# 					m1 += 1
		# 		partner_match += m1
		# 		m2 = 0
		# 		for book_transaction in list_transaction_book:
		# 			for book_buying in partner.get_profile().buy_book.all():
		# 				if book_transaction.book == book_buying.book:
		# 					m2 += 1
		# 		partner_match += m2
		result.append([user,len(book_transactions1),matches,partner_match])
	return result