# coding=utf-8
# __author__ = 'mio'
import logging
import ujson as json

from schema import Schema, Use, Optional, And
from tornado import gen
from tornado.escape import utf8

from models.wholesale_express_logic import ExpressLogic, ExpressIdActionLogic
from tools_lib.dg_utils import get_staff_info
from tools_lib.gtornado.config import MASS_EXPRESS_VALID, MASS_EXPRESS_INVALID, EXPR_STATUS_FINISHED, MASS_EXPRESS_WAIT
from tools_lib.gtornado.escape import schema_utf8
from tools_lib.gtornado.web import AGRequestHandler
from models.mananger import ManagerForceFinishLogic

__UPLOADS__ = "uploads/"

class ManagerForceFinishHandler(AGRequestHandler):
    @gen.coroutine
    def post(self):
        """
        @api {post} /schedule/fe/wholesale_ec/manager/force_finish 电商发货-后台-强行完成订单
        @apiVersion 0.0.1
        @apiName wholesale_ec_manager_ff
        @apiGroup fe_wholesale_ec

        @apiParamExample 请求示例
            POST /schedule/fe/wholesale_ec/manager/force_finish
                body:
                {
                    "courier": {
                        "id": 7773434,
                        "name": "黄映杰"
                    },
                    "numbers": [
                        "000000041234",
                        "000000012555",
                        ......
                    ],
                }

        @apiSuccessExample 成功返回示例
            HTTP/1.1 201 Created
            body:
                {
                    "message": "",
                    "error_code": 0,
                    "content":
                    {
                        "finished": ["000000041234", ......],
                        "unfinished": [
                            {"number": "000000012555", "reason": "找不到订单"},
                            ......
                        ]
                    }
                }

        @apiSuccess (201 Created) {String} message=""
        @apiSuccess (201 Created) {Int} error_code=0
        @apiSuccess (201 Created) {List} content
        @apiSuccess (201 Created) {String} content.finished 已完成列表
        @apiSuccess (201 Created) {String} content.unfinished 未完成列表

        @apiUse ErrorMixin
        """
        query_params = self.get_query_args()
        body_params = json.loads(self.request.body)
        yield self.redirect_request(ManagerForceFinishLogic.post(body_params, query_params))

    def get_schema(self):
        try:
            data = Schema({
                "number": Use(int),
                Optional("shop_id", default=15081): Use(int),
                Optional("skip_fields",
                         default="source,source_order_id,remark,errors,last_error,shop,courier,to_courier,node,vehicle,"
                                 "path,trace,fee,image,urban_driver,callback,is_transferring,pkg_id"
                         ): And(Use(utf8), lambda x: x.split(','))
            }).validate(self.get_query_args())
        except Exception as e:
            logging.error(e.message)
            self.write_parse_args_failed_response(content=e.message)
            return False
        else:
            return data

    @gen.coroutine
    def get(self):
        """
        @api {get} /schedule/fe/wholesale_ec/manager/force_finish 电商发货-后台-查询订单
        @apiVersion 0.0.1
        @apiName wholesale_ec_manager_ff_get
        @apiGroup fe_wholesale_ec

        @apiParamExample 请求示例
            GET /schedule/fe/wholesale_ec/manager/force_finish?number=5252&shop_id=15081

        @apiParam {String} number 电商发货单号, 模糊查询
        @apiParam {String} shop_id=15081 商户id, 默认拼好货id 15081
        @apiParam {String} skip_fields=source,source_order_id,remark,errors,last_error,shop,courier,to_courier,node,vehicle,path,trace,fee,image,urban_driver,callback 不返回的字段

        @apiSuccessExample 成功返回示例
            HTTP/1.1 200 Ok
            body:
                {
                    "content": [
                        {
                            "status": "ADOPTED",
                            "number": "000000046736",
                            "pick_up_time": "2016-01-05T04:10:04Z",
                            "create_time": "2016-01-05T04:10:04Z",
                            "receiver": {
                                "lat": "30.193504",
                                "lng": "120.198532",
                                "tel": "137****5447",
                                "name": "孙某某",
                                "address": "滨和路1799号杭州滨兴学校"
                            },
                            "operator": {
                                "tel": "17705717701",
                                "id": 7792622,
                                "name": "测试黄忠",
                                "avatar": "http://7qnajq.com2.z0.glb.qiniucdn.com/DCE5352E18844B9FC2B6412B3DACD0BD"
                            }
                        }
                    ],
                    "message": "",
                    "error_code": 0
                }

        @apiUse ErrorMixin
        """
        query_params = self.get_schema()
        if query_params is False:
            return
        yield self.redirect_request(ExpressLogic.get(query_params))

# 审核众包配送员订单
class ValidateMassExpressHandler(AGRequestHandler):
    def patch_schema(self):
        try:
            data = Schema({
                "status": lambda x: x in (MASS_EXPRESS_VALID, MASS_EXPRESS_INVALID),
                'number': schema_utf8,
                Optional("remark", default=""): Use(utf8)
            }).validate(self.get_body_args())
        except Exception as e:
            logging.error(e)
            self.write_parse_args_failed_response(content=e.message)
            return False
        else:
            return data

    @gen.coroutine
    def patch(self):
        """
        @api {PATCH} /schedule/fe/wholesale_ec/manager/validate_expr 电商发货-后台-审核众包配送员订单
        @apiVersion 0.0.1
        @apiDescription 注意! 此后台接口需要token
        @apiName wholesale_ec_manager_validate_mass_expr
        @apiGroup fe_wholesale_ec
        @apiUse AuthHeaderMixin

        @apiParamExample 请求示例
            PATCH /schedule/fe/wholesale_ec/manager/validate_expr
            body:{
                "status": "VALID",       //VALID通过, INVALID不通过
                "remark": "备注",
                "number": "000000052126"
            }

        @apiSuccessExample 成功返回示例
            HTTP/1.1 201 CREATED
            body:
                {
                    "content": [
                        {
                            "status": "ADOPTED",
                            "number": "000000046736",
                            "pick_up_time": "2016-01-05T04:10:04Z",
                            "create_time": "2016-01-05T04:10:04Z",
                            "receiver": {
                                "lat": "30.193504",
                                "lng": "120.198532",
                                "tel": "137****5447",
                                "name": "孙某某",
                                "address": "滨和路1799号杭州滨兴学校"
                            },
                            "operator": {
                                "tel": "17705717701",
                                "id": 7792622,
                                "name": "测试黄忠",
                                "avatar": "http://7qnajq.com2.z0.glb.qiniucdn.com/DCE5352E18844B9FC2B6412B3DACD0BD"
                            }
                        }
                    ],
                    "message": "",
                    "error_code": 0
                }

        @apiUse ErrorMixin
        """
        data = self.patch_schema()
        if data is False:
            return

        user_id = yield self.check_staff()
        if not user_id:
            self.write_forbidden_response(content=user_id)
            return

        data['operator'] = {"id": user_id}
        number = data.pop('number', None)
        dg_staff_info = get_staff_info(user_id=user_id)
        if dg_staff_info:
            data['operator'] = {
                "id": user_id,
                "tel": dg_staff_info['phone'],
                "name": dg_staff_info['name'],
                "avatar": dg_staff_info['avatar_qiniu_hash'],
                "staff_num": dg_staff_info['staff_num']
            }

        code, headers, body = yield self.get_response(ExpressIdActionLogic.patch(
            express_number=number, action="validate_info", body_parameters=data
        ))
        self.write_ag_response(code, headers, body)

    @gen.coroutine
    def get(self):
        """
        @api {GET} /schedule/fe/wholesale_ec/manager/validate_expr 电商发货-后台-获取众包配送员订单
        @apiVersion 0.0.1
        @apiDescription 注意! 此后台接口需要token
        @apiName wholesale_ec_manager_get_mass_expr
        @apiGroup fe_wholesale_ec
        @apiUse AuthHeaderMixin

        @apiParamExample 请求示例
            GET /schedule/fe/wholesale_ec/manager/validate_expr?status=VALID

        @apiParam {String} mass_valid_status=WAIT 审核状态,WAIT等待审核,VALID通过, INVALID未通过, ","隔开可查询多个状态, 例如"WAIT,VALID"
        @apiParam {String} expect_fields 需要的字段, ","隔开, 如"number,status,create_time"
        @apiParam {String} args 参数与订单查询接口一致, 不验证token

        @apiSuccessExample 成功返回示例
            HTTP/1.1 201 CREATED
            body:
                {
                    "content": [
                        {
                            "status": "ADOPTED",
                            "number": "000000046736",
                            "pick_up_time": "2016-01-05T04:10:04Z",
                            "create_time": "2016-01-05T04:10:04Z",
                            "receiver": {
                                "lat": "30.193504",
                                "lng": "120.198532",
                                "tel": "137****5447",
                                "name": "孙某某",
                                "address": "滨和路1799号杭州滨兴学校"
                            },
                            "operator": {
                                "tel": "17705717701",
                                "id": 7792622,
                                "name": "测试黄忠",
                                "avatar": "http://7qnajq.com2.z0.glb.qiniucdn.com/DCE5352E18844B9FC2B6412B3DACD0BD"
                            },
                            "validate_info": {
                                "operator": {
                                    "id": 7792622,
                                    "tel": "13022222222",
                                    "name": "王小锤",
                                    "avatar": "qiniu.com/test_hash_lsjlkjl2kjl32",
                                    "staff_num": 1000       //   工号
                                },
                                "status": "WAIT",       // 审核状态
                                "remark": ""
                            },
                        }
                    ],
                    "message": "",
                    "error_code": 0
                }

        @apiUse ErrorMixin
        """
        data = self.get_query_args()

        data['express_status'] = EXPR_STATUS_FINISHED
        if not data.get('mass_valid_status'):
            data['mass_valid_status'] = MASS_EXPRESS_WAIT

        code, headers, body = yield self.get_response(ExpressLogic.get(query_parameters=data))
        self.write_ag_response(code, headers, body)


"""
    Program: ALBERT
    Module: module.py

    This program is free software; you can redistribute it and/or
    modify it under the terms of the GNU General Public License
    version 2 as published by the Free Software Foundation.

    This program is distributed in the hope that it will be useful,
    but WITHOUT ANY WARRANTY; without even the implied warranty of
    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
    GNU General Public License for more details.
"""
__author__ = "Ray Jackson"
__copyright__ = "Copyright 2019, Ray Jackson"
__credits__ = []
__license__ = "GNU GPL"
__version__ = "1.0.0"
__maintainer__ = "Ray Jackson"
__email__ = ""
__status__ = "Production"

import logging
import itertools

from abs_events import AbsUpdatable
from cell import Cell

from node import Node
from constants import *

# noinspection PyBroadException
from model_spec import REST_POTENTIAL, DV_INHIBIT

class Module(AbsUpdatable):

    module_count = 0
    _module_counter = itertools.count(1)

    def __init__(self, brain, module_spec, cell_specs):
        """
        A 2D or 3D collection of nodes providing the spatial scaffolding for cells and bridges.
        """
        # keep count of created objects
        Module.module_count = next(self._module_counter)
        # keep reference to host brain
        self.brain = brain
        # module state
        self.key = f'md-{Module.module_count:02d}'
        self.description = module_spec[DESCRIPTION]
        self.dimensions = module_spec[DIMENSIONS]  # dimensions
        self.bias = REST_POTENTIAL + DV_INHIBIT
        self.cells = {}
        self.axons = []
        self.dendrites = []
        self.spines = []
        self.synapses = []
        self.nodes = {}  # dictionary of nodes referenced by location tuples
        # populate the nodes
        self._create_nodes()
        # create cells
        try:
            for cell_type, cell_spec in module_spec[CELLS].items():
                self.make_cells(cell_type, cell_spec, cell_specs)
        except:
            logging.warning(f"*** Warning: No cells in model: {self.key}")
        # create spines to connect axons and dendrites in each node
        for node in self.nodes.values():
            node.connect()

    def on_state_dump(self):
        return {
            self.key: {
                DESCRIPTION: self.description,
                DIMENSIONS: self.dimensions,
                BIAS: self.bias,
                CELLS: {key: cell.on_state_dump() for key, cell in self.cells.items()},
                AXONS: {axon.key: axon.on_state_dump() for axon in self.axons},
                DENDRITES: {dendrite.key: dendrite.on_state_dump() for dendrite in self.dendrites},
                SPINES: {spine.key: spine.on_state_dump() for dendrite in self.dendrites for spine in dendrite.spines},
                SYNAPSES: {spine.synapse.key: spine.synapse.on_state_dump() for dendrite in self.dendrites for spine in dendrite.spines},
                NODES: {key: node.on_state_dump() for key, node in self.nodes.items()},
            }
        }

    def on_content_request(self):
        return [str(self.key)] + [
            node.on_content_request() for node in self.nodes.values() if len(node.on_content_request()) > 1
        ]

    def make_cells(self, cell_type, cell_spec, cell_specs):
        for location in cell_spec[LOCATIONS]:
            cell = Cell(self, location, cell_specs[cell_type])
            self.cells[cell.key] = cell
            self.nodes[location].cells[cell.key] = cell
            # reference dendrites from the nodes that they cross
            self.add_dendrites_to_nodes(cell)
            # reference axons from the nodes that they cross
            self.add_axons_to_nodes(cell)

    def add_axons_to_nodes(self, cell):
        for axon in cell.axons:
            self.axons.append(axon)
            for loc in axon.locations:
                self.nodes[loc].axons.append(axon)

    def add_dendrites_to_nodes(self, cell):
        for dendrite in cell.dendrites:
            self.dendrites.append(dendrite)
            for spine in dendrite.spines:
                self.spines.append(spine)
            for location in dendrite.locations:
                self.nodes[location].dendrites.append(dendrite)

    def on_tick(self):
        cell_set = set([])
        cells_done = set([])
        cell_next_set = set([])
        # starting with sensors, activate cell network in breadth-first order
        for cell in self.cells.values():
            if cell.cell_type == SENSOR:
                cell_set.add(cell)
        while len(cell_set) > 0:
            # update this layer
            for cell in cell_set:
                # avoid infinite loop if cells feedback into network
                # feedback is incorporated in next pass through the network
                if cell not in cells_done:
                    cell.on_tick()
                    cells_done.add(cell)
                # add child cells to next_set
                for child in cell.children:
                    cell_next_set.add(child)
            # set up for next layer
            cell_set = cell_next_set
            cell_next_set = set([])
        # update nodes
        self.nodes_on_tick()

    def nodes_on_tick(self):
        dims = len(self.dimensions)
        if dims == 2:
            for row in range(self.dimensions[0]):
                for col in range(self.dimensions[1]):
                    self.nodes[(row, col)].on_tick()
        elif dims == 3:
            for row in range(self.dimensions[0]):
                for col in range(self.dimensions[1]):
                    for layer in range(self.dimensions[2]):
                        self.nodes[(row, col, layer)].on_tick()
        else:
            raise ValueError("%s dimensions provided. Only 2 and 3 dimensions are supported." % dims)

    def _create_nodes(self):
        dims = len(self.dimensions)
        if dims == 2:
            self.nodes = {(i, j): Node((i, j), self)
                          for j in range(self.dimensions[1])
                          for i in range(self.dimensions[0])}
        elif dims == 3:
            self.nodes = {(i, j, k): Node((i, j, k), self)
                          for k in range(self.dimensions[2])
                          for j in range(self.dimensions[1])
                          for i in range(self.dimensions[0])}
        else:
            raise ValueError("%s dimensions provided. Only 2 and 3 dimensions are supported." % dims)


import pandas as pd
import os
import pickle
import click
from setup import get_models, get_train_test_split
from data import read_processed_data
from evaluate_models import read_cross_validation_results
from sklearn.metrics import classification_report, confusion_matrix, accuracy_score

path = os.path.dirname(os.path.abspath(__file__))
print(path)

def select_model(cv_results_df):
    # Model selection based on highest mean accuracy
    cv_results_means = cv_results_df.mean()
    cv_results_max = max(cv_results_means)
    selected_model = cv_results_means[cv_results_means == cv_results_max]
    selected_model_name = list(selected_model.index)[0]
    selected_model_mean = list(selected_model)[0]
    print("Model selected based on max mean CV results")
    print("%s is selected with mean CV results %f" %
          (selected_model_name, selected_model_mean))
    return selected_model_name

def prepare_model(best_model):
    models = get_models()
    mod = list(filter(lambda x: x[0] == best_model, models))[0]
    return mod[1]

def validate_model(model, dframe):
    X_train, X_validation, y_train, y_validation = get_train_test_split(dframe)
    model.fit(X_train, y_train)
    predictions = model.predict(X_validation)
    print("Accuracy scores: %f" % accuracy_score(y_validation, predictions))
    print(confusion_matrix(y_validation, predictions))
    print(classification_report(y_validation, predictions))
    return(model)

def save_model_fits(model, filename):
    pickle.dump(model, open(filename, 'wb'))

@click.command()
@click.argument('input_file')
@click.argument('output_file')
def main(input_file, output_file):
    cv_results_df = read_cross_validation_results(input_file)
    best_model = select_model(cv_results_df)
    dframe = read_processed_data()
    model = prepare_model(best_model)
    model_fits = validate_model(model, dframe)
    save_model_fits(model_fits, output_file)

if __name__ == '__main__':
    main()


'''
Program with a function that requests the current weather in Belfast
from OpenWeatherMap.
'''
import json
from urllib import request
import time
import json

def getCurrentWeather(weatherType, city="Belfast"):
   '''
   This function allows you to request the current
   - temperature
   - description
   - wind
   - pressure
   '''

   currentWeather = get_json(city="Belfast")

   weatherInfo = ""

   if weatherType == "temperature":
       weatherInfo = str(currentWeather.get("main").get("temp"))
   elif weatherType == "pressure":
       weatherInfo = str(currentWeather.get("main").get("pressure"))
   elif weatherType == "humidity":
       weatherInfo = str(currentWeather.get("main").get("humidity"))
   elif weatherType == "description":
       weatherInfo = currentWeather.get("weather")[0].get("description")
   elif weatherType == "wind":
       weatherInfo = str(currentWeather.get("wind").get("deg")) + " degrees, " + str(currentWeather.get("wind").get("speed"))  + "mph"

   return(weatherInfo)

def get_json(city="Belfast"):

   # Get appid from config file
   with open("/home/pi/python_scripts/enviroproject/config.json", "r") as f:
    config = json.load(f)
    appid = config["WEATHERAPI"]["APPID"]

   weatherUrl = f"http://api.openweathermap.org/data/2.5/weather?q={city},uk&units=metric&appid={appid}"
   httpResponse = request.urlopen(weatherUrl)
   currentWeather = json.load(httpResponse)
   return currentWeather

def get_sunrise(city="Belfast"):
   currentWeather = get_json(city)
   sunrise =  str(currentWeather.get("sys").get("sunrise"))
   sunrise_time = time.strftime('%H:%M:%S', time.localtime(int(sunrise)))
   return(sunrise_time)

def get_sunset(city="Belfast"):
   currentWeather = get_json(city)
   sunset =  str(currentWeather.get("sys").get("sunset"))
   sunset_time = time.strftime('%H:%M:%S', time.localtime(int(sunset)))
   return(sunset_time)

if __name__ == "__main__":

    my_list = "humidity temperature description wind pressure".split(" ")
    for item in my_list:
       print(getCurrentWeather(item))

    temperature = float(getCurrentWeather("temperature", "Belfast"))

    if temperature >=20:
        print("Go camping on cousin's farm for a month")
    elif temperature >= 15:
        print("Go for a bike ride")
    else:
        print("cry")
    print(f"Sunrise = {get_sunrise()}")

    print(f"Sunset = {get_sunset()}")


import numpy as np # linear algebra
import re
import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)
from sklearn.metrics import log_loss
import matplotlib.pyplot as plt
from nltk import word_tokenize, pos_tag
from nltk.corpus import wordnet
from nltk.stem import WordNetLemmatizer
from nltk.corpus import stopwords
from nltk.stem import SnowballStemmer
from string import punctuation
import nltk
from collections import Counter
from sklearn.model_selection import train_test_split
from gensim.models import word2vec
import Levenshtein
import xgboost as xgb

def word_match_share(row):
    global stops
    q1words = {}
    q2words = {}
    for word in str(row['question1']).lower().split():
        if word not in stops:
            q1words[word] = 1
    for word in str(row['question2']).lower().split():
        if word not in stops:
            q2words[word] = 1
    if len(q1words) == 0 or len(q2words) == 0:
        # The computer-generated chaff includes a few questions that are nothing but stopwords
        return 0
    shared_words_in_q1 = [w for w in q1words.keys() if w in q2words]
    shared_words_in_q2 = [w for w in q2words.keys() if w in q1words]
    R = (len(shared_words_in_q1) + len(shared_words_in_q2))/(len(q1words) + len(q2words))
    return R

def text_to_wordlist(text, remove_stop_words=True, stem_words=False):
    # Clean the text, with the option to remove stop_words and to stem words.
    global stops
    text = re.sub(r"[^A-Za-z0-9]", " ", text)
    text = re.sub(r"what's", "", text)
    text = re.sub(r"What's", "", text)
    text = re.sub(r"\'s", " ", text)
    text = re.sub(r"\'ve", " have ", text)
    text = re.sub(r"can't", "cannot ", text)
    text = re.sub(r"n't", " not ", text)
    text = re.sub(r"I'm", "I am", text)
    text = re.sub(r" m ", " am ", text)
    text = re.sub(r"\'re", " are ", text)
    text = re.sub(r"\'d", " would ", text)
    text = re.sub(r"\'ll", " will ", text)
    text = re.sub(r"60k", " 60000 ", text)
    text = re.sub(r" e g ", " eg ", text)
    text = re.sub(r" b g ", " bg ", text)
    text = re.sub(r"\0s", "0", text)
    text = re.sub(r" 9 11 ", "911", text)
    text = re.sub(r"e-mail", "email", text)
    text = re.sub(r"\s{2,}", " ", text)
    text = re.sub(r"quikly", "quickly", text)
    text = re.sub(r" usa ", " America ", text)
    text = re.sub(r" USA ", " America ", text)
    text = re.sub(r" u s ", " America ", text)
    text = re.sub(r" uk ", " England ", text)
    text = re.sub(r" UK ", " England ", text)
    text = re.sub(r"india", "India", text)
    text = re.sub(r"switzerland", "Switzerland", text)
    text = re.sub(r"china", "China", text)
    text = re.sub(r"chinese", "Chinese", text)
    text = re.sub(r"imrovement", "improvement", text)
    text = re.sub(r"intially", "initially", text)
    text = re.sub(r"quora", "Quora", text)
    text = re.sub(r" dms ", "direct messages ", text)
    text = re.sub(r"demonitization", "demonetization", text)
    text = re.sub(r"actived", "active", text)
    text = re.sub(r"kms", " kilometers ", text)
    text = re.sub(r"KMs", " kilometers ", text)
    text = re.sub(r" cs ", " computer science ", text)
    text = re.sub(r" upvotes ", " up votes ", text)
    text = re.sub(r" iPhone ", " phone ", text)
    text = re.sub(r"\0rs ", " rs ", text)
    text = re.sub(r"calender", "calendar", text)
    text = re.sub(r"ios", "operating system", text)
    text = re.sub(r"gps", "GPS", text)
    text = re.sub(r"gst", "GST", text)
    text = re.sub(r"programing", "programming", text)
    text = re.sub(r"bestfriend", "best friend", text)
    text = re.sub(r"dna", "DNA", text)
    text = re.sub(r"III", "3", text)
    text = re.sub(r"the US", "America", text)
    text = re.sub(r"Astrology", "astrology", text)
    text = re.sub(r"Method", "method", text)
    text = re.sub(r"Find", "find", text)
    text = re.sub(r"banglore", "Banglore", text)
    text = re.sub(r" J K ", " JK ", text)

    # Remove punctuation from text
    text = ''.join([c for c in text if c not in punctuation])

    # Optionally, remove stop words
    if remove_stop_words:
        text = text.split()
        text = [w for w in text if not w in stops]
        text = " ".join(text)

    # Optionally, shorten words to their stems
    if stem_words:
        text = text.split()
        stemmer = SnowballStemmer('english')
        stemmed_words = [stemmer.stem(word) for word in text]
        text = " ".join(stemmed_words)

    # Return a list of words
    return text

def clean_data_frame(row):
    row['question1'] = text_to_wordlist(str(row['question1']))
    row['question2'] = text_to_wordlist(str(row['question2']))
    return row

def get_weight(count, eps=10000, min_count=2):
    if count < min_count:
        return 0
    else:
        return 1 / (count + eps)

def tfidf_word_match_share(row):
    global stops, weights, counts, words
    q1words = {}
    q2words = {}
    for word in str(row['question1']).lower().split():
        if word not in stops:
            q1words[word] = 1
    for word in str(row['question2']).lower().split():
        if word not in stops:
            q2words[word] = 1
    if len(q1words) == 0 or len(q2words) == 0:
        # The computer-generated chaff includes a few questions that are nothing but stopwords
        return 0
    shared_weights = [weights.get(w, 0) for w in q1words.keys() if w in q2words] + [weights.get(w, 0) for w in q2words.keys() if w in q1words]
    total_weights = [weights.get(w, 0) for w in q1words] + [weights.get(w, 0) for w in q2words]
    R = np.sum(shared_weights) / np.sum(total_weights)
    return R

def word2vec_similarity_match(row):
    global stops, w2vmodel
    qcnt = 0
    scnt = 0
    for word1 in str(row['question1']).lower().split():
        if word1 not in stops:
            for word2 in str(row['question2']).lower().split():
                if word2 not in stops:
                    qcnt += 1
                    if word1 in w2vmodel and word2 in w2vmodel:
                        scnt += w2vmodel.wv.similarity(word1, word2)
    if qcnt == 0:
        return 0
    return scnt / qcnt

def word2vec_distance_match(row):
    global stops, w2vmodel
    qcnt = 0
    scnt = 0
    for word1 in str(row['question1']).lower().split():
        if word1 not in stops:
            for word2 in str(row['question2']).lower().split():
                if word2 not in stops:
                    qcnt += 1
                    if word1 in w2vmodel and word2 in w2vmodel:
                        scnt += w2vmodel.wv.distance(word1, word2)
                    else:
                        scnt += 1.0
    if qcnt == 0:
        return 1
    return scnt / qcnt

def edit_distance(row):
    ans = Levenshtein.distance(str(row['question1']), str(row['question2']))
    if ans > 150:
        ans = 150
    return ans

def edit_ratio(row):
    ans = Levenshtein.ratio(str(row['question1']), str(row['question2']))
    return ans

def pre_process_for_datas():
    df_train = pd.read_csv('../input/train.csv')
    print(df_train.head())

    df_train = df_train.apply(clean_data_frame, axis=1, raw=True)
    df_train.to_csv('../input/train_processed.csv', index=False)
    print(df_train.head())

    df_test = pd.read_csv('../input/test.csv')
    print(df_test.head())
    df_test = df_test.apply(clean_data_frame, axis=1, raw=True)
    df_test.to_csv('../input/test_processed.csv', index=False)
    print(df_test.head())

def build_corpus(data):
    corpus = []
    for col in ['question1', 'question2']:
        for sentence in data[col].iteritems():
            word_list = str(sentence[1]).lower().split(" ")
            corpus.append(word_list)
    return corpus

def print_plot(data, df_train, title, filename=None):
    plt.figure(figsize=(15, 5))
    plt.hist(data[df_train['is_duplicate'] == 0], bins=20, density=True, label='Not Duplicate')
    plt.hist(data[df_train['is_duplicate'] == 1], bins=20, density=True, alpha=0.7, label='Duplicate')
    plt.legend()
    plt.title(title, fontsize=15)
    plt.xlabel('word_match_share', fontsize=15)
    if filename is None or filename == '':
        plt.show()
    else:
        plt.savefig(filename)

if __name__ == '__main__':
    #    pre_process_for_datas()

    global stops, words, counts, weights, w2vmodel
    stops = set(stopwords.words("english"))
    df_train = pd.read_csv('../input/train_processed.csv')
    df_test = pd.read_csv('../input/test_processed.csv')

    corpus_train = build_corpus(df_train)
    corpus_test = build_corpus(df_test)
    corpus = corpus_train + corpus_test
    w2vmodel = word2vec.Word2Vec(corpus, size=100, window=20, min_count=500, workers=4)

    train_word_match = df_train.apply(word_match_share, axis=1, raw=True)
    print_plot(train_word_match, df_train, 'Label distribution over word_match_share', '1.png')

    train_qs = pd.Series(df_train['question1'].tolist() + df_train['question2'].tolist()).astype(str)
    test_qs = pd.Series(df_test['question1'].tolist() + df_test['question2'].tolist()).astype(str)
    words = (" ".join(train_qs)).lower().split()
    counts = Counter(words)
    weights = {word: get_weight(count) for word, count in counts.items()}
    tfidf_train_word_match = df_train.apply(tfidf_word_match_share, axis=1, raw=True)
    print_plot(tfidf_train_word_match, df_train, 'Label distribution over tfidf_word_match_share', '2.png')

    train_w2v_similarity_match = df_train.apply(word2vec_similarity_match, axis=1, raw=True)
    print_plot(train_w2v_similarity_match, df_train, 'Label distribution over w2v similarity match', '3.png')

    train_w2v_distance_match = df_train.apply(word2vec_distance_match, axis=1, raw=True)
    print_plot(train_w2v_distance_match, df_train, 'Label distribution over w2v distance match', '4.png')

    train_edit_distance = df_train.apply(edit_distance, axis=1, raw=True)
    print_plot(train_edit_distance, df_train, 'Label distribution over edit distance', '5.png')

    train_edit_ratio = df_train.apply(edit_ratio, axis=1, raw=True)
    print_plot(train_edit_ratio, df_train, 'Label distribution over edit ratio', '6.png')

    x_train_lstm = pd.read_csv('../input/lstm_train.csv')
    x_test_lstm = pd.read_csv('../input/lstm_test.csv')

    del x_train_lstm['id']
    del x_test_lstm['test_id']

    x_train = pd.DataFrame()
    x_test = pd.DataFrame()
    x_train['word_match'] = train_word_match
    x_train['tfidf_word_match'] = tfidf_train_word_match
    x_train['w2v_similarity_match'] = train_w2v_similarity_match
    x_train['w2v_distance_match'] = train_w2v_distance_match
    x_train['edit_distance'] = train_edit_distance
    x_train['edit_ratio'] = train_edit_ratio
    x_test['word_match'] = df_test.apply(word_match_share, axis=1, raw=True)
    x_test['tfidf_word_match'] = df_test.apply(tfidf_word_match_share, axis=1, raw=True)
    x_test['w2v_similarity_match'] = df_test.apply(word2vec_similarity_match, axis=1, raw=True)
    x_test['w2v_distance_match'] = df_test.apply(word2vec_distance_match, axis=1, raw=True)
    x_test['edit_distance'] = df_test.apply(edit_distance, axis=1, raw=True)
    x_test['edit_ratio'] = df_test.apply(edit_ratio, axis=1, raw=True)
    y_train = df_train['is_duplicate'].values

    x_train = pd.merge(x_train, x_train_lstm, how='inner', left_index=True, right_index=True)
    x_test = pd.merge(x_test, x_test_lstm, how='inner', left_index=True, right_index=True)

    pos_train = x_train[y_train == 1]
    neg_train = x_train[y_train == 0]

    # Now we oversample the negative class
    # There is likely a much more elegant way to do this...
    len_ori_neg_train = len(neg_train)
    p = 0.1293
    scale = ((len(pos_train) / (len(pos_train) + len(neg_train))) / p) - 1
    while scale > 1:
        neg_train = pd.concat([neg_train, neg_train[:len_ori_neg_train]])
        scale -= 1
    neg_train = pd.concat([neg_train, neg_train[:int(scale * len_ori_neg_train)]])
    print(len(pos_train) / (len(pos_train) + len(neg_train)))

    x_train = pd.concat([pos_train, neg_train])
    y_train = (np.zeros(len(pos_train)) + 1).tolist() + np.zeros(len(neg_train)).tolist()
    del pos_train, neg_train
    x_train, x_valid, y_train, y_valid = train_test_split(x_train, y_train, test_size=0.1, random_state=4242)

    # Set our parameters for xgboost
    params = {}
    params['objective'] = 'binary:logistic'
    params['eval_metric'] = 'logloss'
    params['eta'] = 0.015
    params['max_depth'] = 4

    d_train = xgb.DMatrix(x_train, label=y_train)
    d_valid = xgb.DMatrix(x_valid, label=y_valid)

    watchlist = [(d_train, 'train'), (d_valid, 'valid')]
    bst = xgb.train(params, d_train, 10, watchlist, early_stopping_rounds=50, verbose_eval=10)

    for i in range(2, 1000):
        iters = i * 10
        print('training ' + str(iters) + ':')
        bst_2 = xgb.train(params, d_train, 10, watchlist, early_stopping_rounds=50, verbose_eval=10, xgb_model=bst)
        bst = bst_2

        d_test = xgb.DMatrix(x_test)
        p_test = bst.predict(d_test)

        sub = pd.DataFrame()
        sub['test_id'] = df_test['test_id']
        sub['is_duplicate'] = p_test
        sub.to_csv('simple_xgb_6_iter' + str(iters) + '_lstm.csv', index=False)


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Sat Dec 22 09:37:21 2018

@author: osezeiyore
"""

import gurobipy as gb
import math
import pandas as pd
#import numpy as np

#%% Compact Bender's decomposition

UBB=0
UB = math.inf
UB2 = math.inf
LB = -math.inf
eps = 1e-5
a_down = -1e18

sum_mast = pd.DataFrame(columns = ['g1','g2','g3','WT','theta1','theta2','a','obj'])
sum_sub1 = pd.DataFrame(columns = ['UB','G31','SP1','SH1','g1_k','g2_k','g3_k','WT_k','theta1_k','theta2_k','thetaRT1','thetaRT2','rho1','rho2','rho3','rhoWT','rhot1','rhot2'])
sum_sub2 = pd.DataFrame(columns = ['UB','G31','SP1','SH1','g1_k','g2_k','g3_k','WT_k','theta1_k','theta2_k','thetaRT1','thetaRT2','rho1','rho2','rho3','rhoWT','rhot1','rhot2'])
sum_sub3 = pd.DataFrame(columns = ['UB','G31','SP1','SH1','g1_k','g2_k','g3_k','WT_k','theta1_k','theta2_k','thetaRT1','thetaRT2','rho1','rho2','rho3','rhoWT','rhot1','rhot2'])

k = 0

# initialize master problem
mast = gb.Model()
mast.setParam( 'OutputFlag', False )

g1_status = mast.addVar(vtype=gb.GRB.BINARY)
g2_status = mast.addVar(vtype=gb.GRB.BINARY)
g3_status = mast.addVar(vtype=gb.GRB.BINARY)

g1 = mast.addVar(lb=0, ub=50)
g2 = mast.addVar(lb=0, ub=110)
g3 = mast.addVar(lb=0, ub=100)
WT = mast.addVar(lb=0, ub =50)
theta1 = mast.addVar()
theta2 = mast.addVar()
a = mast.addVar(lb = -gb.GRB.INFINITY)

obj_m = (10*g1+25*g2+35*g3) + a
mast.setObjective(obj_m,gb.GRB.MINIMIZE)
mast.addConstr(g1+g2+1000*(theta2-theta1)==200)
mast.addConstr(g3+WT-1000*(theta2-theta1)==0)
mast.addConstr(g1 >= g1_status*20)
mast.addConstr(g1 <= g1_status*50)
mast.addConstr(g2 >= g2_status*20)
mast.addConstr(g2 <= g2_status*110)
mast.addConstr(g3 >= g3_status*10)
mast.addConstr(g3 <= g3_status*100)
mast.addConstr(a >= a_down)

mast.update()
g1_f=0
g2_f=0
g3_f=0
WT_f=0
theta1_f=0
theta2_f=0

while abs(UB-LB)>eps:
    #---------------------------  SUB PROBLEM 1  ------------------------------
    sub = gb.Model()
    sub.setParam( 'OutputFlag', False )

    # variables (i)
    g1_k = sub.addVar(lb = 0, ub=50)
    g2_k = sub.addVar(lb = 0, ub=110)
    g3_k = sub.addVar(lb=0, ub=100)
    WT_k = sub.addVar(lb = 0, ub=50)
    theta1_k = sub.addVar()
    theta2_k = sub.addVar()

    # varibles
    G31 = sub.addVar(lb=-45, ub=45)
    SP1 = sub.addVar(lb=0, ub=50)
    SH1 = sub.addVar(lb=0, ub=200)
    thetaRT1 = sub.addVar(lb=-gb.GRB.INFINITY, ub=gb.GRB.INFINITY)
    thetaRT2 = sub.addVar(lb=-gb.GRB.INFINITY, ub=gb.GRB.INFINITY)

    sub.setObjective(0.2*(35*G31+200*SH1),gb.GRB.MINIMIZE)

    # constraints
    if (g3_f>0):
        sub.addConstr(g3_k+G31 >=10)
    if (g3_f==0):
        sub.addConstr(g3_k+G31 ==0)
    sub.addConstr(0 <= g3_k+G31)
    sub.addConstr(g3_k+G31 <= 100, name='pi')
    sub.addConstr(SP1 <= 50, name='mu')
    sub.addConstr(SH1 <= 200, name='o')
    sub.addConstr(g3_k+G31+WT_k<= 200)
    sub.addConstr(G31+(50-WT_k-SP1)-1000*((thetaRT2-thetaRT1)-(theta2_k-theta1_k))==0)
    sub.addConstr(SH1+1000*((thetaRT2-thetaRT1)-(theta2_k-theta1_k))==0)
    sub.addConstr(g1_k == g1_f, name='rho1')
    sub.addConstr(g2_k == g2_f, name='rho2')
    sub.addConstr(g3_k == g3_f, name='rho3')
    sub.addConstr(WT_k == WT_f, name='rhoWT')
    sub.addConstr(theta1_k == theta1_f, name='rhot1')
    sub.addConstr(theta2_k == theta2_f, name='rhot2')
    sub.addConstr(1000*(thetaRT2-thetaRT1) <=200)
    sub.addConstr(1000*(thetaRT2-thetaRT1) >=-200)
    sub.update()
    sub.optimize()
    UB1 = sub.objVal
    sum_sub1.loc[k,'UB'] = UB1
    sum_sub1.loc[k,'G31'] = G31.x
    sum_sub1.loc[k,'SP1'] = SP1.x
    sum_sub1.loc[k,'SH1'] = SH1.x
    sum_sub1.loc[k,'g1_k'] = g1_k.x
    sum_sub1.loc[k,'g2_k'] = g2_k.x
    sum_sub1.loc[k,'g3_k'] = g3_k.x
    sum_sub1.loc[k,'WT_k'] = WT_k.x
    sum_sub1.loc[k,'theta1_k'] = theta1_k.x
    sum_sub1.loc[k,'theta2_k'] = theta2_k.x
    sum_sub1.loc[k,'thetaRT1'] = thetaRT1.x
    sum_sub1.loc[k,'thetaRT2'] = thetaRT2.x
    sum_sub1.loc[k,'rho1'] = sub.getConstrByName('rho1').Pi
    sum_sub1.loc[k,'rho2'] = sub.getConstrByName('rho2').Pi
    sum_sub1.loc[k,'rho3'] = sub.getConstrByName('rho3').Pi
    sum_sub1.loc[k,'rhoWT'] = sub.getConstrByName('rhoWT').Pi
    sum_sub1.loc[k,'rhot1'] = sub.getConstrByName('rhot1').Pi
    sum_sub1.loc[k,'rhot2'] = sub.getConstrByName('rhot2').Pi
    del sub

    #---------------------------  SUB PROBLEM 2  ------------------------------

    sub2 = gb.Model()
    sub2.setParam( 'OutputFlag', False )

     # variables (i)
    g1_k = sub2.addVar(lb = 0, ub=50)
    g2_k = sub2.addVar(lb = 0, ub=110)
    g3_k = sub2.addVar(lb=0, ub=100)
    WT_k = sub2.addVar(lb = 0, ub=50)
    theta1_k = sub2.addVar()
    theta2_k = sub2.addVar()

    # varibles
    G31 =sub2.addVar(lb=-45, ub=45)
    SP1 = sub2.addVar(lb=0, ub=50)
    SH1 = sub2.addVar(lb=0, ub=200)
    thetaRT1 = sub2.addVar()
    thetaRT2 = sub2.addVar()

    sub2.setObjective(0.5*(35*G31+200*SH1),gb.GRB.MINIMIZE)

    # constraints

    if (g3_f>0):
        sub2.addConstr(g3_k+G31 >=10)
    if (g3_f==0):
        sub2.addConstr(g3_k+G31 ==0)
    sub2.addConstr(0 <= g3_k+G31)
    sub2.addConstr(g3_k+G31 <= 100, name='pi')
    sub2.addConstr(SP1 <= 22, name='mu')
    sub2.addConstr(SH1 <= 200, name='o')
    sub2.addConstr(g3_k+G31+WT_k<= 200)
    sub2.addConstr(G31+(22-WT_k-SP1)-1000*((thetaRT2-thetaRT1)-(theta2_k-theta1_k))==0)
    sub2.addConstr(SH1+1000*((thetaRT2-thetaRT1)-(theta2_k-theta1_k))==0)
    sub2.addConstr(g1_k == g1_f, name='rho1')
    sub2.addConstr(g2_k == g2_f, name='rho2')
    sub2.addConstr(g3_k == g3_f, name='rho3')
    sub2.addConstr(WT_k == WT_f, name='rhoWT')
    sub2.addConstr(theta1_k == theta1_f, name='rhot1')
    sub2.addConstr(theta2_k == theta2_f, name='rhot2')
    sub2.addConstr(1000*(thetaRT2-thetaRT1) <=200)
    sub2.addConstr(1000*(thetaRT2-thetaRT1) >=-200)
    sub2.update()
    sub2.optimize()
    UB2 = sub2.objVal
    sum_sub2.loc[k,'UB'] = UB2
    sum_sub2.loc[k,'G31'] = G31.x
    sum_sub2.loc[k,'SP1'] = SP1.x
    sum_sub2.loc[k,'SH1'] = SH1.x
    sum_sub2.loc[k,'g1_k'] = g1_k.x
    sum_sub2.loc[k,'g2_k'] = g2_k.x
    sum_sub2.loc[k,'g3_k'] = g3_k.x
    sum_sub2.loc[k,'WT_k'] = WT_k.x
    sum_sub2.loc[k,'theta1_k'] = theta1_k.x
    sum_sub2.loc[k,'theta2_k'] = theta2_k.x
    sum_sub2.loc[k,'thetaRT1'] = thetaRT1.x
    sum_sub2.loc[k,'thetaRT2'] = thetaRT2.x
    sum_sub2.loc[k,'rho1']  = sub2.getConstrByName('rho1').Pi
    sum_sub2.loc[k,'rho2']  = sub2.getConstrByName('rho2').Pi
    sum_sub2.loc[k,'rho3']  = sub2.getConstrByName('rho3').Pi
    sum_sub2.loc[k,'rhoWT'] = sub2.getConstrByName('rhoWT').Pi
    sum_sub2.loc[k,'rhot1'] = sub2.getConstrByName('rhot1').Pi
    sum_sub2.loc[k,'rhot2'] = sub2.getConstrByName('rhot2').Pi
    del sub2

    #---------------------------  SUB PROBLEM 3  ------------------------------

    sub3 = gb.Model()
    sub3.setParam( 'OutputFlag', False )

    # variables (i)
    g1_k = sub3.addVar(lb = 0, ub=50)
    g2_k = sub3.addVar(lb = 0, ub=110)
    g3_k = sub3.addVar(lb=0, ub=100)
    WT_k = sub3.addVar(lb = 0, ub=50)
    theta1_k = sub3.addVar()
    theta2_k = sub3.addVar()

    # varibles
    G31 =sub3.addVar(lb=-45, ub=45)
    SP1 = sub3.addVar(lb=0, ub=50)
    SH1 = sub3.addVar(lb=0, ub=200)
    thetaRT1 = sub3.addVar()
    thetaRT2 = sub3.addVar()

    sub3.setObjective(0.3*(35*G31+200*SH1),gb.GRB.MINIMIZE)

    # constraints
    if (g3_f>0):
        sub3.addConstr(g3_k+G31 >=10)
    if (g3_f==0):
        sub3.addConstr(g3_k+G31 ==0)
    sub3.addConstr( 0 <= g3_k+G31)
    sub3.addConstr(g3_k+G31 <= 100, name='pi')
    sub3.addConstr(SP1 <= 10, name='mu')
    sub3.addConstr(SH1 <= 200, name='o')
    sub3.addConstr(G31+(10-WT_k-SP1)-1000*((thetaRT2-thetaRT1)-(theta2_k-theta1_k))==0)
    sub3.addConstr(SH1+1000*((thetaRT2-thetaRT1)-(theta2_k-theta1_k))==0)
    sub3.addConstr(g3_k+G31+WT_k<= 200)
    sub3.addConstr(g1_k == g1_f, name='rho1')
    sub3.addConstr(g2_k == g2_f, name='rho2')
    sub3.addConstr(g3_k == g3_f, name='rho3')
    sub3.addConstr(WT_k == WT_f, name='rhoWT')
    sub3.addConstr(theta1_k == theta1_f, name='rhot1')
    sub3.addConstr(theta2_k == theta2_f, name='rhot2')
    sub3.addConstr(1000*(thetaRT2-thetaRT1) <=200)
    sub3.addConstr(1000*(thetaRT2-thetaRT1) >=-200)
    sub3.update()
    sub3.optimize()
    UB3 = sub3.objVal
    sum_sub3.loc[k,'UB'] = UB3
    sum_sub3.loc[k,'G31'] = G31.x
    sum_sub3.loc[k,'SP1'] = SP1.x
    sum_sub3.loc[k,'SH1'] = SH1.x
    sum_sub3.loc[k,'g1_k'] = g1_k.x
    sum_sub3.loc[k,'g2_k'] = g2_k.x
    sum_sub3.loc[k,'g3_k'] = g3_k.x
    sum_sub3.loc[k,'WT_k'] = WT_k.x
    sum_sub3.loc[k,'theta1_k'] = theta1_k.x
    sum_sub3.loc[k,'theta2_k'] = theta2_k.x
    sum_sub3.loc[k,'thetaRT1'] = thetaRT1.x
    sum_sub3.loc[k,'thetaRT2'] = thetaRT2.x
    sum_sub3.loc[k,'rho1'] = sub3.getConstrByName('rho1').Pi
    sum_sub3.loc[k,'rho2'] = sub3.getConstrByName('rho2').Pi
    sum_sub3.loc[k,'rho3'] = sub3.getConstrByName('rho3').Pi
    sum_sub3.loc[k,'rhoWT'] = sub3.getConstrByName('rhoWT').Pi
    sum_sub3.loc[k,'rhot1'] = sub3.getConstrByName('rhot1').Pi
    sum_sub3.loc[k,'rhot2'] = sub3.getConstrByName('rhot2').Pi
    del sub3

    #--------------------------- MASTER PROBLEM  ------------------------------

    mast.addConstr(a >=0.2*(35*sum_sub1.loc[k,'G31'] + 200*sum_sub1.loc[k,'SH1'])+ 0.5*(35*sum_sub2.loc[k,'G31']\
                        + 200*sum_sub2.loc[k,'SH1'])+ 0.3*(35*sum_sub3.loc[k,'G31'] + 200*sum_sub3.loc[k,'SH1'])\
                        + (sum_sub1.loc[k,'rho1']+sum_sub2.loc[k,'rho1']+sum_sub3.loc[k,'rho1'])*(g1-(sum_sub1.loc[k,'g1_k']+sum_sub2.loc[k,'g1_k']+sum_sub3.loc[k,'g1_k'])*(1/3))\
                        + (sum_sub1.loc[k,'rho2']+sum_sub2.loc[k,'rho2']+sum_sub3.loc[k,'rho2'])*(g2-(sum_sub1.loc[k,'g2_k']+sum_sub2.loc[k,'g2_k']+sum_sub3.loc[k,'g2_k'])*(1/3))\
                        + (sum_sub1.loc[k,'rho3']+sum_sub2.loc[k,'rho3']+sum_sub3.loc[k,'rho3'])*(g3-(sum_sub1.loc[k,'g3_k']+sum_sub2.loc[k,'g3_k']+sum_sub3.loc[k,'g3_k'])*(1/3))\
                        + (sum_sub1.loc[k,'rhoWT']+sum_sub2.loc[k,'rhoWT']+sum_sub3.loc[k,'rhoWT'])*(WT-(sum_sub1.loc[k,'WT_k']+sum_sub2.loc[k,'WT_k']+sum_sub3.loc[k,'WT_k'])*(1/3))\
                        + (sum_sub1.loc[k,'rhot1']+sum_sub2.loc[k,'rhot1']+sum_sub3.loc[k,'rhot1'])*(theta1-(sum_sub1.loc[k,'theta1_k']+sum_sub2.loc[k,'theta1_k']+sum_sub3.loc[k,'theta1_k'])*(1/3))\
                        + (sum_sub1.loc[k,'rhot2']+sum_sub2.loc[k,'rhot2']+sum_sub3.loc[k,'rhot2'])*(theta2-(sum_sub1.loc[k,'theta2_k']+sum_sub2.loc[k,'theta2_k']+sum_sub3.loc[k,'theta2_k'])*(1/3)))

    mast.update()
    mast.optimize()

    g1_f = g1.x
    g2_f = g2.x
    g3_f = g3.x
    WT_f = WT.x
    theta1_f = theta1.x
    theta2_f = theta2.x
    LB = a.x
    UB=sum_sub1.loc[k,'UB']+sum_sub2.loc[k,'UB']+sum_sub3.loc[k,'UB']

    sum_mast.loc[k,'g1'] = g1_f
    sum_mast.loc[k,'g2'] = g2_f
    sum_mast.loc[k,'g3'] = g3_f
    sum_mast.loc[k,'WT'] = WT_f
    sum_mast.loc[k,'theta1'] = theta1_f
    sum_mast.loc[k,'theta2'] = theta2_f
    sum_mast.loc[k,'a'] = LB
    sum_mast.loc[k,'obj'] = mast.objVal
    sum_mast.loc[k,'UB']=UB

    k += 1

print(sum_sub1)
print(sum_sub2)
print(sum_sub3)
print(sum_mast)


import urllib.request
import json
import sys

#Coded by Ashish
#https://github.com/Ashish0804/IMDb2BBCode

#Paste your api key here
api_key=''
base_url='http://www.omdbapi.com/?apikey='+api_key
imbd_tag=str(sys.argv[1])
imbd_url='https://www.imdb.com/title/'+imbd_tag
url=base_url+'&i='+imbd_tag
data = json.load(urllib.request.urlopen(url))
out_file=open("imbd2bb_out.txt","w")

out_template='''
[center][size=150]{}({})[/size][/center]

[center][url={}]{}[/url][/center]

[center][img]{}[/img][/center]

[color=#FF8000][b]Release Date[/b][/color]: {}

[color=#FF8000][b]Viewer Rating[/b][/color]: {}

[color=#FF8000][b]Runtime[/b][/color]: {}

[color=#FF8000][b]Genre[/b][/color]: {}

[color=#FF8000][b]Actors[/b][/color]: {}

[color=#FF8000][b]Plot[/b][/color]: {}
'''
if data['Response']=='False':
	print(data['Error'])
else:
	out_text = out_template.format(data['Title'],data['Year'],imbd_url,imbd_tag,data['Poster'],data['Released'],data['Rated'],data['Runtime'],data['Genre'],data['Actors'],data['Plot'])
	out_file.write(out_text)
	out_file.close()
	print("Done Writing")


import os
import pickle
from imutils import paths
import cv2
import dlib
import face_recognition

image_path = '/Volumes/projects/advanced_project/recognizer_server/recognizer_server/datasets'
model_path = '/Volumes/projects/advanced_project/recognizer_server/recognizer_server/'

imagepaths = list(paths.list_images(image_path))
imagePaths = []
for i in imagepaths:
	x = i.split(os.path.sep)[-1]
	if not x.startswith('.'):
		imagePaths.append(i)

knownEncodings = []
knownNames = []

print ("Encoding images...")
for (i, image) in enumerate(imagePaths):
	name = image.split(os.path.sep)[-2]
	image = cv2.imread(image)
	rgb_image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
	boxes = face_recognition.face_locations(rgb_image, model='cnn')
	encodings = face_recognition.face_encodings(rgb_image, boxes)
	for encoding in encodings:
		knownEncodings.append(encoding)
		knownNames.append(name)

print("[INFO] serializing encodings...")
data = {"encodings": knownEncodings, "names": knownNames}
with open(model_path + 'test.pickle', "wb") as file:
	file.write(pickle.dumps(data))


import cv2
import numpy as np
from matplotlib import pyplot as plt

#此文件试图给图片补上竖着的线

#0 黑色  255 白色
img = cv2.imread('test.png', 0)
ret, img = cv2.threshold(img, 127, 255, cv2.THRESH_BINARY)

shape = img.shape
print(shape)
row = shape[0]
col = shape[1]
count = 0

#获取图片的横线
lines = []
for i in range(row):
    count = 0
    for j in range(col):
        px = img[i, j]
        if px == 0:
            count += 1
    if count > int(3*row/2):
        lines.append(i)
print("一共找到如下横线"+str(lines))
linesNum = len(lines)

xBegin = 0
for i in range(col):
    if img[lines[0], i] == 0:
        xBegin = i
        break

xEnd = 0
for i in range(col-1, 0, -1):
    if img[lines[0], i] == 0:
        xEnd = i
        break
print("x"+str(xBegin)+" x" +str(xEnd))
#
myList = []
last = -1
for i in range(row):
    count = 0
    for j in range(col):
        px = img[i, j]
        if px == 0:
            count += 1
    if count <= 2:
        myList.append(i)
print(myList)

newList = []
i = 0
while i < len(myList):
    temp = []
    for j in range(myList[i], myList[-1]):
        if j in myList:
            temp.append(j)
            i += 1
        else:
            i -= 1
            break
    i += 1
    newList.append(temp)
print(newList)

#
# #最左边的边处理 最右边的边处理
# newList.remove(newList[0])
# newList.remove(newList[-1])
# cv2.line(img, (xBegin, lines[0]), (xBegin, lines[-1]), (0, 0, 0), 1)
# cv2.line(img, (xEnd, lines[0]), (xEnd, lines[-1]), (0, 0, 0), 1)
#

def judge(list, num):
    for item in list:
        if abs(num - item) <= 5:
            return False
    return True

for i in newList:
    if len(i) > 7:
        num = int((i[0] + i[-1])/2)
        print(num)
        if not judge(lines, num):
            continue
        cv2.line(img, (xBegin, num), (xEnd, num), (0, 0, 0), 1)

cv2.imshow('image', img)
#cv2.imwrite("hhh.png", img)
cv2.waitKey(0)
cv2.destroyAllWindows()


# coding=utf-8
# !/usr/bin/python3

from common.base_config import BaseConfig
from configs import config, urls
import requests
from requests.exceptions import *

class RequestUrl(BaseConfig):
    """
    url请求类，继承配置基类
    """

    def __init__(self):
        super(RequestUrl, self).__init__()
        self.__code = config.CODE  # 正确的响应状态码
        self.__timeout = config.TIMEOUT  # 请求超时的路径
        self.__url_dicts = urls.URL_LISTS
        self.urls = {x: self.__url_dicts[x] for x in range(len(self.__url_dicts))}  # 请求连接转换为字典
        self.urls_len = len(self.urls)  # 个数统计
        self.res_true = 0  # 正常请求个数统计
        self.res_false = 0  # 异常请求个数统计
        self.res_tf = []  # 请求是否成功 1 or 0
        self.msgs_false = ''  # 异常的访问信息发送
        self.false_url_index = ''  # 记录访问失败的url索引
        self.false_code = ''  # 请求失败状态记录，包含状态码、超时连接
        self.false_url_list = None  # 失败的url索引list
        self.false_code_list = None  # 失败的状态信息list
        self.__request_url()
        self.false_url_list = self.__false_url_list()  # 失败的url索引list
        self.false_code_list = self.__false_code_list()  # 失败的状态信息list

    def __request_url(self):
        """
        循环请求需要用到的方法和数据
        :return:
        """
        _res = requests.Session()
        for i in range(len(self.urls)):
            dict_url = self.urls[i][1]  # url
            dict_name = self.urls[i][0]  # 名称
            connect = self.__timeout['connect']  # 连接超时时间
            read = self.__timeout['read']  # 读取超时时间
            try:
                res = _res.get(dict_url, timeout=(connect, read))
                res_code = res.status_code  # 获取状态码
                if res_code in self.__code:
                    self.res_tf.append('1')  # 访问成功1
                    self.res_true += 1  # 正常访问个数
                else:
                    msg = ".   请求错误：GetCodeError：" + str(res_code)
                    self.__handle_res_msgs(dict_name, dict_url, i, msg)

            except ConnectTimeout:
                msg = ".   连接超时：ConnectTimeout: " + str(connect) + "s"
                self.__handle_res_msgs(dict_name, dict_url, i, msg)

            except ReadTimeout:
                msg = ".   读取超时：ReadTimeout: " + str(read) + "s"
                self.__handle_res_msgs(dict_name, dict_url, i, msg)

            except ConnectionError as e:
                txt = 'connection:'
                if txt in str(e):
                    txt_error = str(e).split('connection:')[1][:-4]
                    error = None
                else:
                    txt_error = 'OtherError'
                    error = e
                msg = ".   连接错误：ConnectionError:" + txt_error
                self.__handle_res_msgs(dict_name, dict_url, i, msg, error=error)

            except Exception as e:
                msg = '.   其他错误信息，请查看日志文件'
                self.__handle_res_msgs(dict_name, dict_url, i, msg, error=e)

    def __handle_res_msgs(self, name, url, index, msg, *, error=None):
        """
        请求信息处理
        :return:
        """
        self.res_tf.append('0')  # 访问失败0
        msg2 = name + ' ' + url + msg  # 失败状态码记录
        self.false_code += str(index) + msg + '\n'  # 请求失败状态记录，状态码
        self.msgs_false += str(self.res_false + 1) + '、' + msg2 + '\n'  # 失败信息叠加
        self.res_false += 1  # 失败访问个数
        self.false_url_index += str(index) + '\n'  # 访问失败的url索引
        if error is None:
            self.logger.error('%-90s%s%30s' % (url, msg[4:-1], name))
        else:
            self.logger.error('%-90s%s%30s' % (url, '其他错误信息：' + str(error), name))

    def __false_url_list(self):
        """
        将失败url索引记录转换为list
        :return: 处理后的list
        """
        return self.false_url_index.split('\n')[:-1]

    def __false_code_list(self):
        """
        将失败请求的信息记录转换为list
        :return: 处理后的list
        """
        return self.false_code.split('\n')[:-1]

if __name__ == '__main__':
    run = RequestUrl()
    # run.run_request()
    print(run.res_true)
    print(run.res_false)
    print(run.res_tf)
    print(run.msgs_false)
    print(run.false_url_index)
    print(run.false_url_list)
    print(run.false_code)
    print(run.false_code_list)


from kafka import KafkaProducer
import json

KAFKA_IP = '192.168.5.70'
KAFKA_PORT = '9092'
TOPIC = 'test'
DB = 'backup-data.json'

# produce json messages

producer = KafkaProducer(bootstrap_servers=[KAFKA_IP + ':' + KAFKA_PORT],
                         value_serializer=lambda m: json.dumps(m).encode('ascii'))

with open(DB, 'r') as db:
    for i, line in enumerate(db):
        if i < 1 or i > 100:
            continue
        print(line[:-2])
        data = json.loads(line[:-2])
        print(data)
        producer.send(TOPIC, data)

# block until all async messages are sent
producer.flush()


import logging
import os
import torch
# import torchvision.models as models
# from networks.net import VGGNet

def to_tuple_str(str_first, gpu_num, str_ind):
    if gpu_num > 1:
        tmp = '('
        for cpu_ind in range(gpu_num):
            tmp += '(' + str_first + '[' + str(cpu_ind) + ']' + str_ind + ',)'
            if cpu_ind != gpu_num - 1:
                tmp += ', '
        tmp += ')'
    else:
        tmp = str_first + str_ind
    return tmp

def load_s_model(model, ckpt_path):
    print("------------")
    if os.path.exists(ckpt_path):
        model.load_state_dict(torch.load(ckpt_path))
        print('load ' + str(ckpt_path))
    else:
        print('=> no student checkpoint find')
    print('------------')

def load_t_model(model, ckpt_path):
    print("------------")
    if os.path.exists(ckpt_path):
        model.load_state_dict(torch.load(ckpt_path))
        print('load' + str(ckpt_path))
    else:
        print('=> no teacher checkpoint find')
    print('------------')

def print_model_parm_nums(model, string):
    b = []
    for param in model.parameters():
        b.append(param.numel())
    print(string + ': Number of params: %.2fM' % (sum(b) / 1e6))

def l2(feat):
    return (((feat ** 2).sum(dim=1)) ** 0.5).reshape(feat.shape[0], 1, feat.shape[1], feat.shape[2]) + 1e-8

def similarity(feature):
    feature = feature.float()
    tmp = l2(feature).detach()
    feature = feature / tmp
    feature = feature.reshape(feature.shape[0], feature.shape[1], -1)
    return torch.einsum('icm, icn -> imn', [feature, feature])

def sim_dis_compute(f_s, f_t):
    sim_err = ((similarity(f_t) - similarity(f_s)) ** 2) / ((f_t.shape[-1] * f_t.shape[-2]) ** 2) / f_t.shape[0]
    sim_dis = sim_err.sum()
    return sim_dis

def build_graph(f1, f2):
    h = f1.size(1)
    w = f1.size(2)
    m = f1.size(0)
    n = f2.size(0)
    g = []
    for i in range(m):
        r = []
        for j in range(n):
            r.append(torch.sum(torch.mul(f1[i], f2[j])) / (h * w))
        g.append(r)
    return g

# if __name__ == '__main__':
#     net_arch16 = [64, 64, 'M', 128, 128, 'M', 256, 256, 256, 'M', 512, 512,
#                   512, 'M', 512, 512, 512, 'M5', "FC1", "FC2", "FC"]
#     teacher = VGGNet(net_arch16, 10)
#     print(torch.nn.Sequential(*list(teacher.children())[:]))
#     model = models.vgg16(pretrained=False)
#     print(torch.nn.Sequential(*list(model.children())[:]))


import pygame #게임을 만들기 위한 pygame 라이브러리
import random #랜덤 라이브러리

class Stage:
    def __init__(self, width, height, foodlist):
        self.width = width # 가로 줄 음식의 개수
        self.height = height # 세로 줄 음식의 개수
        self.x = (800-((width*60)+((width-1)*5)))//2  # 1행 1열에 올 음식의 초기 x값
        self.y = (800-((height*60)+((height-1)*5)))//2-45 # 1행 1열에 올 음식의 초기 y값
        self.foodlist = foodlist # 음식 종류의 리스트를 self.foodlist에 할당
        self.condition = True # self.condition = True 변수 선언

        self.stage = [] # 빈 리스트 생성
        tmp = [] # 빈 리스트 생성
        for j in range(height): # 인자로 넘겨받은 height만큼 반복
            for i in range(width): # 인자로 넘겨받은 width만큼 반복
                tmp.append(Grid(self.x+i*65,self.y+j*65,j,i,random.choice(self.foodlist))) # self.x(초기x) + i*65, self.y(초기y) i*65, j, i, random한 self.foodlist를 받은 Grid를 tmp에 추가
            self.stage.append(tmp) # self.stage에 tmp를 추가
            tmp = [] #tmp 초기화

    def update(self): # stage update 메소드
        li = [] # 빈 리스트li 생성
        for i in self.stage: # i에 self.stage의 요소를 넘겨 줌 (위에 의해 Grid를 담은 리스트를 넘겨줌)
            for j in i: # j에 Grid를 담은 리스트의 요소 하나하나를 넘겨 줌
                if j: # j가 존재하는지 검사
                    li.append(j.update()) # j는 Grid 객체이므로 j는 Grid객체의 update메소드를 수행하고 이를 li에 추가한다
        return li # li 반환

    def fallen(self):
        for j in range(len(self.stage[0])): # 0부터 (self.stage[0]의 길이-1)까지 오름차순으로 하나씩 j에 할당
            for i in range(len(self.stage)): # 0부터 (self.stage의 요소(리스트) 수-1)까지 오름차순으로 하나씩 i에 할당 (세로방향을 위한 2중 for문)
                if not self.stage[i][j] and i-1>=0 and self.stage[i-1][j]: # 2열~끝열 중, self.stage[i][j](음식)이 존재하지 않고, self.stage[i-1][j](음식)이 존재 할 때
                    self.stage[i-1][j].fall() # self.stage[i-1][j](Grid 객체)은 fall메소드가 적용된다. (fall메소드는 class Grid의 메소드임)
                    self.stage[i][j],self.stage[i-1][j] = self.stage[i-1][j],self.stage[i][j] # self.stage[i][j],self.stage[i-1][j]를 서로 바꿔준다.
        for j in range(len(self.stage[0])): # 0부터 (self.stage[0]의 길이-1)까지 오름차순으로 하나씩 j에 할당
            for i in range(len(self.stage)-1): # 0부터 (self.stage의 요소(리스트) 수-2)까지 오름차순으로 하나씩 i에 할당
                if self.stage[i][j] and not self.stage[i+1][j]: # self.stage[i][j]가 존재하고, self.stage[i+1][j]가 존재하지 않을 때
                    self.fallen() # 객체 자신은 fallen메소드를 호출한다
        return self.stage # self.stage를 리턴한다

    def check(self):
        for i in range(len(self.stage)): # 0부터 (self.stage의 요소(리스트) 수-1)까지 오름차순으로 하나씩 i에 할당
            for j in range(len(self.stage[0])): # 0부터 (self.stage[0]의 길이-1)까지 오름차순으로 하나씩 j에 할당 (가로방향을 위한 2중 for문)
                if type(self.stage[i][j])!=int and (self.stage[i][j].move_y or self.stage[i][j].move_x): # self.stage[i][j]의 타입이 int가 아니면서 self.stage[i][j].move_y 또는 self.stage[i][j].move_x가 0이 아닐 때
                    return "moving",0,0 # ('moving', 0, 0)을 반환
        dli = self.stage # dli에 self.stage 할당 (얕은 복사)
        result = [] # 빈 리스트 result생성
        for i in range(len(dli)): # 0부터 (dli(self.stage의 요소(리스트)) 수-1)까지 오름차순으로 하나씩 i에 할당
            for j in range(len(dli[0])): # 0부터 (dli[0](self.stage[0])의 길이-1)까지 오름차순으로 하나씩 j에 할당
                if i+2<len(dli) and (len(set(list(map(str,[dli[i][j],dli[i+1][j],dli[i+2][j]]))))==1)\
                     and dli[i][j]!= 0: # i+2<len(dil)을 통해 맨위 음식부터 맨아래에서 3번째 까지 검사, dli[i][j],dli[i+1][j],dli[i+2][j] 각각을 map을 통해 str로 변환 후 리스트로 변환, 그 후 집합으로 변환시키고, 집합의 길이가 1인지 검사(dli[i][j],dli[i+1][j],dli[i+2][j]이 모두 같은지 검사 즉, 3x1 모양이 같은지 검사)
                    if not [i,j] in result: # 리스트 result에 리스트 [i,j]가 없는지 검사(없으면 참)
                        result.append([i,j]) # 리스트 result에 리스트 [i,j]를 추가
                    if not [i+1,j] in result: # 리스트 result에 리스트 [i+1,j]가 없는지 검사(없으면 참)
                        result.append([i+1,j]) # 리스트 result에 리스트 [i+1,j]를 추가
                    if not [i+2,j] in result: # 리스트 result에 리스트 [i+2,j]가 없는지 검사(없으면 참)
                        result.append([i+2,j]) # 리스트 result에 리스트 [i+2,j]를 추가
                if i+1<len(dli) and j+1<len(dli[0])\
                    and(len(set(list(map(str,[dli[i][j],dli[i+1][j],dli[i][j+1],dli[i+1][j+1]]))))==1)\
                     and dli[i][j]!=0: # 위와 비슷한 원리로 2x2 모양의 같은 음식이 있는지 검사
                    if not [i,j] in result: # 리스트 result에 리스트 [i,j]가 없는지 검사(없으면 참)
                        result.append([i,j]) # 리스트 result에 리스트 [i,j]를 추가
                    if not [i+1,j] in result: # 리스트 result에 리스트 [i+1,j]가 없는지 검사(없으면 참)
                        result.append([i+1,j]) # 리스트 result에 리스트 [i+1,j]를 추가
                    if not [i,j+1] in result: # 리스트 result에 리스트 [i,j+1]가 없는지 검사(없으면 참)
                        result.append([i,j+1]) # 리스트 result에 리스트 [i,j+1]를 추가
                    if not [i+1,j+1] in result: # 리스트 result에 리스트 [i+1,j+1]가 없는지 검사(없으면 참)
                        result.append([i+1,j+1]) # 리스트 result에 리스트 [i+1,j+1]를 추가
                if j+2<len(dli[0]) and (len(set(list(map(str,[dli[i][j],dli[i][j+1],dli[i][j+2]]))))==1)\
                     and dli[i][j]!=0: # 위와 비슷한 원리로 1x3모양이 같은지 검사
                    if not [i,j] in result: # 리스트 result에 리스트 [i,j]가 없는지 검사(없으면 참)
                        result.append([i,j]) # 리스트 result에 리스트 [i,j]를 추가
                    if not [i,j+1] in result: # 리스트 result에 리스트 [i,j+1]가 없는지 검사(없으면 참)
                        result.append([i,j+1]) # 리스트 result에 리스트 [i,j+1]를 추가
                    if not [i,j+2] in result: # 리스트 result에 리스트 [i,j+2]가 없는지 검사(없으면 참)
                        result.append([i,j+2]) # 리스트 result에 리스트 [i,j+2]를 추가
        for i,j in result: # result의 요소인 리스트 0번째 인덱스의 값과 1번째 인덱의 값을 i,j에 각각 할당
            dli[i][j] = 0 # dli[i][j]의 값을 0으로 바꿔 줌 (self.stage[i][j] = 0과 같음)
        score = len(result) # score에 result 안의 요소(리스트) 수를 score에 할당
        return (score, dli, result) # 튜플 형태로 score, dli, result의 값을 보내줌

    def fill(self):
        for i in range(len(self.stage)): # 0부터 (self.stage의 요소(리스트) 수-1)까지 오름차순으로 하나씩 i에 할당
            for j in range(len(self.stage[0])):  # 0부터 (self.stage[0]의 길이-1)까지 오름차순으로 하나씩 j에 할당
                if self.stage[i][j]!=0 and self.stage[i][j].move_y: # self.stage[i][j]의 값이 0이 아니면서 sel.stage[i][j].move_y의 값이 0이 아닌지 검사
                    return # 종료? ~~
        for i in sorted(list(range(len(self.stage))),reverse=True): # (self.stage의 요소(리스트) 수-1)부터 0까지 내림차순으로 i에 할당
            for j in sorted(list(range(len(self.stage[0]))),reverse=True): # (self.stage[0]의 길이 -1)부터 0까지 내림차순으로 j에 할당
                if self.stage[i][j]==0: # 맨오른쪽 맨아래 요소부터 self.stage[i][j]의 값이 0인지 검사
                    self.stage[i][j] = Grid(self.x+j*65,self.y-65,i,j,random.choice(self.foodlist)) # self.x(초기x) + i*65, self.y(초기y) i*65, i, j, random한 self.foodlist를 받은 Grid를 tmp에 추가 (i, j가 아니라 j, i 순임)
                    self.stage[i][j].move_y += abs(self.y+(i+1)*65 - self.y) # self.stage[i][j]의 값은 Grid 객체이므로 그 객체의 인스턴스 변수 move_y의 값에 절댓값 self.y+(i+1)*65 - self.y의 값을 더한다. (거꾸로 검사하기에 절댓값을 이용한 계산)

    def test(self,pos):
        x,y = pos # test메소드의 pos라는 인자의 0번째 인덱스, 1번째 인덱스 값을 각각 x,y에 할당
        for i in range(len(self.stage)): # 0부터 (self.stage의 요소(리스트) 수-1)까지 오름차순으로 하나씩 i에 할당
            for j in range(len(self.stage[0])): # 0부터 (self.stage[0]의 길이-1)까지 오름차순으로 하나씩 j에 할당
                if type(self.stage[i][j])==Grid and \
                    self.stage[i][j].rect.x <x< self.stage[i][j].rect.x+60 and \
                   self.stage[i][j].rect.y <y< self.stage[i][j].rect.y+60: # self.stage[i][j]의 타입이 class Grid인지, 인자로 받은 pos의 x,y가 모든 Grid객체 중 하나라도 내부에 속하는지 검사
                   return self.stage[i][j].pos_x,self.stage[i][j].pos_y # (self.stage[i][j].pos_x,self.stage[i][j].pos_y) 튜플 반환
        return (-10,-10) # 위 조건 불만족 시 (-10, -10) 튜플 반환

    def swap(self,i,j):
        if i[0]==j[0]: # 튜플i와 튜플j 각각의 0번째 인덱스가 같은지 검사
            value_x = self.stage[i[0]][i[1]].rect.x - self.stage[j[0]][j[1]].rect.x # self.stage[i[0]][i[1]].rect.x - self.stage[j[0]][j[1]].rect.x 의 값을 value_x에 할당
            self.stage[i[0]][i[1]].move_x = value_x*-1 # value_x*(-1)의 값을 self.stage[i[0]][i[1]](Grid 객체)의 move_x 변수에 할당
            self.stage[j[0]][j[1]].move_x = value_x # value_x의 값을 self.stage[j[0]][j[1]](Grid 객체)의 move_x 변수에 할당 (self.stage[i[0]][i[1]]가 아님에 유의)
            if value_x <0: # value_x의 값이 0보다 작은지 검사
                self.stage[i[0]][i[1]].pos_y += 1 # self.stage[i[0]][i[1]].pos_y에 1을 더해줌
                self.stage[j[0]][j[1]].pos_y -= 1 # self.stage[j[0]][j[1]].pos_y에 1을 빼줌 (i가 아님에 유의)
            else: # value_x의 값이 0보다 클 때
                self.stage[i[0]][i[1]].pos_y -= 1 # 위와 반대로 self.stage[i[0]][i[1]].pos_y에 1을 빼줌
                self.stage[j[0]][j[1]].pos_y += 1 # self.stage[j[0]][j[1]].pos_y에 1을 더해줌 (i가 아님에 유의)
            self.stage[i[0]][i[1]],self.stage[j[0]][j[1]] = self.stage[j[0]][j[1]],self.stage[i[0]][i[1]] # self.stage[i[0]][i[1]]과 self.stage[j[0]][j[1]]의 위치를 서로 바꿔줌
        elif i[1]==j[1]: # 튜플i와 튜플j 각각의 1번째 인덱스가 같은지 검사
            value_y = self.stage[i[0]][i[1]].rect.y - self.stage[j[0]][j[1]].rect.y # self.stage[i[0]][i[1]].rect.y - self.stage[j[0]][j[1]].rect.y 의 값을 value_y에 할당
            self.stage[i[0]][i[1]].move_y = value_y*-1 # value_y*(-1)의 값을 self.stage[i[0]][i[1]](Grid 객체)의 move_y 변수에 할당
            self.stage[j[0]][j[1]].move_y = value_y # value_x의 값을 self.stage[j[0]][j[1]](Grid 객체)의 move_y 변수에 할당 (self.stage[i[0]][i[1]]가 아님에 유의)
            if value_y <0: # value_y의 값이 0보다 작은지 검사
                self.stage[i[0]][i[1]].pos_x += 1 # self.stage[i[0]][i[1]].pos_x에 1을 더해줌
                self.stage[j[0]][j[1]].pos_x -= 1 # self.stage[j[0]][j[1]].pos_x에 1을 빼줌 (i가 아님에 유의)
            else: # value_y의 값이 0보다 클 때
                self.stage[i[0]][i[1]].pos_x -= 1 # 위와 반대로 self.stage[i[0]][i[1]].pos_x에 1을 빼줌
                self.stage[j[0]][j[1]].pos_x += 1 # self.stage[j[0]][j[1]].pos_x에 1을 더해줌 (i가 아님에 유의)
            self.stage[i[0]][i[1]],self.stage[j[0]][j[1]] = self.stage[j[0]][j[1]],self.stage[i[0]][i[1]] # self.stage[i[0]][i[1]]과 self.stage[j[0]][j[1]]의 위치를 서로 바꿔줌

class Grid(pygame.sprite.Sprite):
    def __init__(self,x,y,pos_x,pos_y,t): # 인자로 x, y, pos_x, pos_y, t를 받음
        pygame.sprite.Sprite.__init__(self)
        self.surface = pygame.image.load('img/{}.png'.format(t)) # 스프라이트에 이미지 t를 지정한다. (위의 코드에서 알 수 있듯이, t는 random.choice(self.foodlist)이다.)
        self.type = t # t라는 음식의 이름을 self.type에 할당
        self.rect = self.surface.get_rect() # surface의 초기x, 초기y, 가로,세로 길이를 담은 객체를 self.rect에 할당
        self.pos_x = pos_x # 인자로 받은 pos_x를 self.pos_x에 할당
        self.pos_y = pos_y # 인자로 받은 pos_y를 self.pos_y에 할당
        self.rect.x = x # 인자로 받은 x를 self.rect.x에 할당 (self.rect.x == self.rect[0])
        self.rect[1] = y # 인자로 받은 y를 self.rect.y에 할당 (self.rect.x == self.rect[1])
        self.move_x = 0 # self.move_x의 값을 0으로 초기화 및 선언
        self.move_y = 0 # self.move_y의 값을 0으로 초기화 및 선언

    def fall(self):
        self.move_y += 65 # self.move_y의 값에 65를 더해줌
        self.pos_x += 1 # self.pos_x에 1을 더해줌

    def update(self):
        if self.move_x: # self.move_x 가 0이 아닌지 검사
            if self.move_x > 0: # self.move.x 값이 0보다 큰지 검사
                self.rect.x += 5 # self.rect.x의 값을 5 더해줌
                self.move_x -= 5 # self.move_x의 값을 5 빼줌
            else: # self.move.x 값이 0보다 작을 때
                self.rect.x -= 5 # self.rect.x의 값을 5 빼줌
                self.move_x += 5 # self.move_x의 값을 5 더해줌
        if self.move_y: # self.move_y가 0이 아닌지 검사
            if self.move_y > 0: # self.move_y 값이 0보다 큰지 검사
                self.rect.y += 5 # self.rect.y의 값을 5 더해줌
                self.move_y -= 5 # self.move_y의 값을 5 빼줌
            else: # self.move.y 값이 0보다 작을 때
                self.rect.y -= 5 # self.rect.y의 값을 5 빼줌
                self.move_y += 5 # self.move_y의 값을 5 더해줌

        return self.surface, (self.rect.x, self.rect.y) # (self.surface, (self.rect.x, self.rect.y)) 튜플을 반환

    def __repr__(self):
        return self.type #  객체를 인간이 이해할 수 있는 평문으로 '표현'

    def __str__(self):
        return self.type # 서로 다른 자료형 간에 인터페이스를 제공하기 위함

class Button(pygame.sprite.Sprite):
    def __init__(self,x,y,width=80,height=50,color=(0,0,0),text='',textcolor=(255,255,255),func=None):
        pygame.sprite.Sprite.__init__(self)
        self.surface = pygame.Surface([width,height]) #width = 80, height = 50인 surface 생성
        self.width,self.height,self.color = width,height,color # 객체의 width,height,color을 설정. 각각은 80,50,(0,0,0) (참고로 (0,0,0)은 검정이다.)
        self.surface.fill(color) # color로 surface를 채운다
        self.rect = self.surface.get_rect() # 객체의 (가로길이,세로길이) 튜플을 self.rect에 할당한다
        self.rect.x = x # surface의 초기 x위치를 지정한다
        self.rect.y = y # surface의 초기 y위치를 지정한다
        self.font = pygame.font.Font('font/cabin.ttf',20) # cabin.ttf폰트를 지정한다
        self.text = self.font.render(text,True,textcolor) # text의 속성을 설정한다 인수 각각은 text, antialias, color을 의미한다
        self.func = func # func를 self.func에 할당한다 default of func는 None이다

    def update(self):
        return self.surface, (self.rect.x,self.rect.y),\
            self.text, (self.rect.x+10,((self.rect.y*2+self.height)//2)-12) # (self.surface, (self.rect.x,self.rect.y), self.text, (self.rect.x+10,((self.rect.y*2+self.height)//2)-12))를 반환한다

    def over(self):
        pass

    def isClick(self,pos):
        if self.rect.x < pos[0] < self.rect.x+self.width: # 마우스로부터 받은 x위치가 self.rect.x와 self.rect.x+self.width 사이에 있는지 검사
            if self.rect.y < pos[1] < self.rect.y+self.height: # 마우스로부터 받은 y위치가 self.rect.y와 self.rect.y+self.width 사이에 있는지 검사
                if self.func: # self.func가 None이 아닌지 검사
                    self.func() # self.func 메소드 실행
                return True # 두 조건문이 모두 만족되면 True를 반환
        return False # 두 조건문 중 하나라도 만족되지 않으면 False를 반환

class Game:
    def __init__(self):
        pygame.init() # pygame 초기화
        pygame.mixer.init() # pygame mixer (사운드 관련) 초기화
        pygame.display.set_caption("CookingCrush!") # 실행창 이름을 설정
        self.screen = pygame.display.set_mode((800, 960)) # 게임 화면의 크기를 800x960으로 설정
        self.clock = pygame.time.Clock() # self.clock.tick과 같이 시간 관련 메소드를 편리하게 사용할 수 있도록 함
        self.stage = Stage(8,9,['h','c','p','o','t','s']) # Stage 객체 생성. 인자는 각각 가로의 음식 개수, 세로의 음식 개수, 음식 객체들의 이름을 모은 리스트 이다.
        self.first = () #첫번째 클릭 상자 좌표
        self.second = () #두번째 클릭 상자 좌표
        self.score = 0 # 처음 score = 0 으로 초기화
        self.resetbutton = Button(280,692,text="RESET",func=self.reset) # resetbutton 객체 생성. 인자는 각각  x좌표, y좌표, text, func 이다
        self.quitbutton = Button(600,692,text="QUIT") # quitbutton 객체 생성. 인자는 각각 x좌표, y좌표, text, func 이다
        self.font = pygame.font.Font('font/cabin.ttf',20) # font 설정
        self.zone = pygame.image.load("img/check.png") # 클릭시 빨간 네모테두리 이미지 설정
        self.sound = [pygame.mixer.Sound('sound/click.wav'),pygame.mixer.Sound('sound/score.wav')] # 사운드 2가지를 묶은 리스트 생성
        pygame.mixer.music.load('sound/b.wav') # main bgm을 로드한다
        pygame.mixer.music.set_volume(1) # 소리 크기 1로 설정 (1이 최대)
        pygame.mixer.Sound.set_volume(self.sound[0],10) # 클릭 소리 크기 1로 설정 (1보다 커도 최대가 1이므로 1로 설정)
        pygame.mixer.music.play(-1) # 무한 반복
        pygame.time.delay(1300) # 로딩시간 설정

    def update(self):
        self.board = pygame.Surface([550,620]) # 가로 550, 세로 620 크기의 surface 생성
        self.score_board = pygame.Surface([130,50]) # 가로 130, 세로 50 크기의 surface 생성
        self.board.fill((140,75,10)) # (140, 75, 10) 의 색깔로 self.board를 채움 (괄호 안은 R,G,B 순이다)
        self.score_board.fill([140,75,10]) # (140, 75, 10)의 색깔로 score_board를 채움
        self.screen.fill((255,255,255)) # 메인화면 screen의 색을 흰색으로 채움

        self.screen.blit(pygame.Surface([564,634]),(113,48)) # surface(겉 테두리) 그리기
        self.screen.blit(self.board,(120,55)) # self.board 그리기
        self.screen.blit(self.score_board,(113,692)) # self.score_board 그리기

        tmp = self.stage.update() # [Grid 객체, (x좌표, y좌표)]들이 요소로 들어있는 리스트를 tmp에 할당
        for i in tmp: # tmp의 요소(리스트)를 i에 하나씩 줌
            self.screen.blit(i[0],i[1]) # i[0](Grid 객체), i[1](x좌표,y좌표)임을 이용하여 screen에 Grid객체를 그림

        tmp = self.resetbutton.update() # resetbutton의 (테두리 객체, (x좌표, y좌표), 텍스트 객체, (x좌표, y좌표))의 튜플을 tmp에 넘겨 줌
        self.screen.blit(tmp[0],tmp[1]) # resetbutton 테두리 그리기
        self.screen.blit(tmp[2],tmp[3]) # resetbutton 텍스트 그리기
        tmp = self.quitbutton.update() # quitbutton의 (테두리 객체, (x좌표, y좌표), 텍스트 객체, (x좌표, y좌표))의 튜플을 tmp에 넘겨 줌
        self.screen.blit(tmp[0],tmp[1]) # quitbutton 테두리 그리기
        self.screen.blit(tmp[2],tmp[3]) # quitbutton 텍스트 그리기

        if self.first: # self.first이 빈튜플이 아닌지 검사
            self.screen.blit(self.zone,(self.first[0],self.first[1])) # self.zone을 x좌표: self.first[0], y좌표: self.first[1]에 그리기
        if self.second: # self.second가 빈튜플이 아닌지 검사
            self.screen.blit(self.zone,(self.second[0],self.second[1])) # self.zone을 x좌표: self.second[0], y좌표: self.second[1]에 그리기
        self.screen.blit(self.font.render("SCORE : {}".format(self.score),True,(255,255,255)),(120,705)) # self.score을 검정색글씨로 antialias를 적용하여 (x: 120, y: 705)에 그린다
        pygame.display.update() # 파이게임 화면을 업데이트해준다

    def reset(self):
        for i in range(len(self.stage.stage)): # 0부터 (self.stage.stage의 요소(리스트) 수-1)까지 오름차순으로 하나씩 i에 할당
            for j in range(len(self.stage.stage[0])): # 0부터 (self.stage.stage[0]의 길이-1)까지 오름차순으로 하나씩 j에 할당
                self.stage.stage[i][j] = 0 # self.stage.stage[i][j]를 0으로 초기화
        self.score = 0 # self.score을 0으로 초기화

    def main(self):
        run = True # run을 True로 초기화
        self.score = 0 # self.score을 0으로 초기화
        firstc, secondc = (-10,-10),(-10,-10) # firstc와 secondc를 각각 (-10,-10)으로 초기화
        con = False # con을 False로 초기화
        while run: # 처음 초기화 된 run이 True이므로 특별한 일이 생기기 전까지 무한반복
            for event in pygame.event.get(): # 파이게임의 이벤트기능을 쓸 수 있도록 event를 받는다
                if event.type == pygame.QUIT: # 종료버튼을 누르는지 검사
                    run = False # run의 값을 False로 바꿔준다 즉, 무한루프 탈출
                if event.type == pygame.KEYDOWN: # 키보드의 버튼을 누른 상태인지 검사

                    pass
                if event.type == pygame.MOUSEBUTTONDOWN: # 마우스 버튼이 눌려진 상태인지 검사
                    self.resetbutton.isClick(pygame.mouse.get_pos()) # reset 버튼활성화로, 마우스 버튼의 (x좌표: pygame.mouse.get_pos()[0], y좌표: pygame.mouse.get_pos()[1])의 튜플을 isClick의 pos인자로 전달한다 그리고 isClick의 self.func()에 의해 reset 메소드가 실행된다
                    if self.quitbutton.isClick(pygame.mouse.get_pos()): # 마우스 버튼의 (x좌표: pygame.mouse.get_pos()[0], y좌표: pygame.mouse.get_pos()[1])의 튜플을 isClick의 pos인자로 전달한다
                        run = False # run 값을 False로 바꿔준다 즉, 무한루프 탈출
                    firstc = (-10,-10) # firstc를 (-10, -10)으로 초기화
                    firstc = self.stage.test(pygame.mouse.get_pos()) # 마우스 버튼의 (x좌표: pygame.mouse.get_pos()[0], y좌표: pygame.mouse.get_pos()[1])의 튜플을 test의 pos인자로 전달한다 그 후 self.stage.test의 반환값을 firstc에 할당한다
                    if firstc!=(-10,-10): # firstc가 (-10, -10)이 아닐 때 즉, 마우스가 어떤 Grid객체 내부 좌표에 속할 때
                        tmp = self.stage.stage[firstc[0]][firstc[1]] # self.stage.stage[firstc[0]][firstc[1]]을 tmp에 할당
                        self.first = (tmp.rect.x,tmp.rect.y) # (tmp.rect.x,tmp.rect.y)의 튜플을 self.first에 할당 (클릭한 곳의 Grid객체의 x, y좌표)
                        pygame.mixer.Sound.play(self.sound[0]) # 클릭 소리 재생
                if event.type == pygame.MOUSEMOTION and firstc != (-10,-10) : # 마우스가 움직이고 firstc가 (-10, -10)이 아닌지 검사
                    pos = self.stage.test(pygame.mouse.get_pos()) # 마우스 버튼의 (x좌표: pygame.mouse.get_pos()[0], y좌표: pygame.mouse.get_pos()[1])의 튜플을 test의 pos인자로 전달한다 그 후 self.stage.test의 반환값을 pos에 할당한다
                    if pos!=(-10,-10): #pos가 (-10, -10)이 아닌지 검사
                        tmp = self.stage.stage[pos[0]][pos[1]] # self.stage.stage[pos[0]][pos[1]]을 tmp에 할당
                        self.second = (tmp.rect.x,tmp.rect.y) # (tmp.rect.x,tmp.rect.y)의 튜플을 self.second에 할당 (클릭한 곳의 Grid객체의 x, y좌표)
                if event.type == pygame.MOUSEBUTTONUP: # 누른 마우스 버튼이 올라갈 때를 검사
                    secondc = (-10,-10) # secondc를 (-10, -10)으로 초기화
                    secondc = self.stage.test(pygame.mouse.get_pos()) # 마우스 버튼의 (x좌표: pygame.mouse.get_pos()[0], y좌표: pygame.mouse.get_pos()[1])의 튜플을 test의 pos인자로 전달한다 그 후 self.stage.test의 반환값을 secondc에 할당한다

            #교체
            if abs(firstc[0]-secondc[0] + firstc[1]-secondc[1])==1 and self.first and self.second: # 십자가 모양으로 한칸 차이의 Grid객체를 선택했는지 검사 후 first 체크박스와 second 체크박스가 있는지 검사
                pygame.mixer.Sound.play(self.sound[0]) # 클릭 소리 재생
                self.stage.condition = True # self.stage.condition을 True로 초기화
                self.stage.swap(firstc,secondc) # swap함수의 인자 i, j에 각각 firstc, secondc를 할당
                a,b = firstc,secondc # a, b에 각각 firstc, secondc 할당
                con = True # con을 True로 초기화
                firstc, secondc = (-10,-10),(-10,-10) # firstc, secondc를 각각 (-10,-10), (-10,-10)로 초기화
                self.first , self.second = (),() # self.first , self.second를 각각 (), ()로 초기화

            check = self.stage.check() #점수 체크

            #교체후 점수가 획득되면 다시 교체 안함
            if check[0]!=0 and check[0]!="moving": # check[0]가 0이 아닌지 검사하고 "moving"이 아닌지 검사
                if check[0] > 6: # check[0]이 6보다 큰지 검사
                    pygame.mixer.Sound.play(self.sound[1]) # 득점 소리 재생
                elif check[0] > 4: # check[0]이 4보다 큰지 검사
                    pygame.mixer.Sound.play(self.sound[1]) # 득점 소리 재생
                else: # check[0]이 4이하일 때
                    pygame.mixer.Sound.play(self.sound[1]) # 득점 소리 재생
                self.score += check[0] # self.score에 check[0]의 값을 더한다
                con = False # con 값을 False로 바꾼다

            #교체후 점수가 획득되지 않으면 다시 원래자리로
            if con and check[0]==0: # con이 True이고 check[0]가 0인지 검사
                con = False # con의 값을 False로 바꾼다
                self.stage.swap(a,b) # self.stage.swap함수의 인자 i, j에 a, b를 각각 대입한다

            self.stage.fallen() # stage의 fallen메소드 실행
            self.stage.fill() # 스테이지의 fill메소드 실행
            self.update() # update메소드 실행
            self.clock.tick(80) # 프레임 80
        pygame.quit() # pygame 종료

if __name__ == '__main__': # main 스크립트 파일일 때
    game = Game() # game이름의 Game 객체 생성
    game.main() # game 객체를 통한 main 메소드 실행

#swap
#self.stage.stage
#reset


from heapq import heappush, heappop, heapify
from utils import create_node

def index(f, s):
    return next((i for i, x in enumerate(f) if x[1][0] == s), -1)

#problem.initialstate=
def uniform_cost_graph_search(problem):#state parent pathcost
    initial_node = create_node(problem.initials_state, None, 0)
    frontier = [(0, initial_node)]
    explored = []
    n_visits = 0
    while True:
        if not frontier:
            return (None, n_visits)
        else:
            n_visits += 1
            _, node = heappop(frontier)
            state, _,  path_cost,  = node
            explored.append(state)
            if problem.is_goal(state):
                return (node, n_visits)
            else:
                for succ in problem.successors(state):
                    child_cost = path_cost + 1
                    child = create_node(succ, node,  child_cost)
                    #print(succ)
                    #print(child_cost)
                    if succ not in explored:
                        idx = index(frontier, succ)
                        if idx < 0:
                            heappush(frontier, (child_cost, child))
                        else:
                            _, existing = frontier[idx]
                            if existing[2] > child_cost:
                                frontier[idx] = (child_cost, child)
                                heapify(frontier)


"""
Given an integer list where each number represents the number of hops you can make,
determine whether you can reach to the last index starting at index 0.

For example, [2, 0, 1, 0] returns true while [1, 1, 0, 1] returns false.

Solution:
track the visited positions in a set
track the positions you can & will visit in a queue

Stop when stack is empty
Check the visited stack & see if last idx is inside
"""
import collections

def steps(hops):
    visited = set()
    queue = collections.deque()
    queue.append(0)
    visited.add(0)

    while queue:
        idx = queue.popleft()
        value = hops[idx]

        for i in range(1, value + 1):
            next_idx = idx + i
            if next_idx not in visited:
                visited.add(next_idx)
                queue.append(next_idx)
    return len(hops) - 1 in visited

if __name__ == "__main__":
    expect_true = [2, 0, 1, 0]
    assert steps(expect_true)
    expect_false = [1, 1, 0, 1]
    assert steps(expect_false) == False


import pytest
from selenium import webdriver
from selenium.webdriver.common.keys import Keys
from selenium.common.exceptions import NoSuchElementException
# run after test
#@pytest.yield_fixture()

def test_contact():
    driver = webdriver.Chrome('C:/Users/User/Documents/DocDoc/Autotesting/chromedriver.exe')
    driver.get("https://shop.docdoc.ru/")
    driver.find_element_by_link_text("Контакты").click()
    #driver.switch_to.window(driver.window_handles[1])
    assert driver.find_element_by_xpath('//h1')=="Контакты"
    driver.close()

def test_user__agreement():
    driver = webdriver.Chrome('C:/Users/User/Documents/DocDoc/Autotesting/chromedriver.exe')
    driver.get("https://shop.docdoc.ru/")
    try:
        driver.find_element_by_link_text("Пользовательское соглашение").click()
    except :
        print("not found")
    driver.close()


#!/usr/bin/env python
from __future__ import print_function, division
from astropy.table import Table
from astropy import log
import datetime
import os
import argparse
import multiprocessing as mp
import numpy as np
import pandas as pd
import shutil
import subprocess
import time

#LommenResearchGroup imports
import pipeline_utils

#Dom Rowan 2020

desc = """
Pipeline for pulsar analysis
"""

#nicersoft psrpipe wrapper
def run_psrpipe(obsID, par,
				emin=0.25, emax=12,
				min_sun=60,
				keith=True, crab=False):

    log.info("Running pspipe with par " + par)

    if not os.path.isdir(obsID):
        print("obsID not found")
        return -1

    cmd = ['custom_psrpipe.py',
           '--emin', str(emin),
           '--emax', str(emax),
           '--minsun', str(min_sun)]

    #Add par file if it is given
    if par is not None:
        cmd.extend(['--par', par])
    else:
        log.info("No par file input: pulse phase will not be defined")

    if keith:
        cmd.append('--keith')

    #Use this argument to do photonphase with multiprocessing
    if crab:
        cmd.append('--crab')

    cmd.append(obsID)

    subprocess.call(cmd)

#runs full pipe on single observation
def allprocedures(obsID, par,
				  emin=0.25, emax=12,
				  trumpet=True, keith=True,
				  clobber=False, crab=False):

    #Various clobber and path checks
    if ((os.path.isdir("{}_pipe".format(obsID)))
        and (not clobber)):
        return 0
    elif ((os.path.isdir("{}_pipe".format(obsID)))
            and clobber):
        log.info("Removing {}_pipe".format(obsID))
        shutil.rmtree("{}_pipe".format(obsID))

    if not os.path.isdir(obsID):
        log.error("obsID not found")
        return -1

    #Some observations are missing event cl directory
    #Exit here if that's the case
    if not os.path.isdir(obsID+'/xti/event_cl'):
        log.error("no event cl dir for {}".format(obsID))
        return -1

    #Run nicerl2
    if (not pipeline_utils.check_nicerl2(obsID)) or clobber:
        pipeline_utils.run_nicerl2(obsID, trumpet=trumpet, clobber=clobber)

    #Add KP info
    pipeline_utils.run_add_kp(obsID)

    #Run psrpipe
    run_psrpipe(obsID, par,
                emin=emin, emax=emax,
                keith=keith, crab=crab)
    return 1

#Multiprocessing wrapper
def wrapper(par, emin=0.25, emax=12,
			trumpet=True, keith=True, clobber=False,
            crab=False):

    #if we supply table of par info
    if par.endswith('.csv'):
        multiple_pars = True
        log.info("Reading par information from csv")
        df = pd.read_csv(par, keep_default_na=False)
        par_dic = {}
        for i in range(len(df)):
            if df['par'][i] == '':
                continue
            else:
                par_dic[str(df['obsID'][i])] = df['par'][i]
    else:
        multiple_pars = False

    #If we are doing crab we don't want to setup pool here
    if crab:
        #Find observations to pipe
        for f in os.listdir('./'):
            if (os.path.isdir(f)) and (f.isnumeric()):
                #par matching
                if multiple_pars:
                    if f not in list(par_dic.keys()):
                        log.info("No par for {}".format(f))
                        continue
                    else:
                        par = par_dic[f]

                allprocedures(f, par, emin=emin, emax=emax,
                              trumpet=trumpet, keith=keith,
                              clobber=clobber, crab=crab)

    else: #setup multiprocessing pool
        #Create pool
        pool = mp.Pool(processes=mp.cpu_count()+2)
        jobs = []

        #Find observations to pipe
        for f in os.listdir("./"):
            if (os.path.isdir(f)) and (f.isnumeric()):

                #par check for crab
                if multiple_pars:
                    if f not in list(par_dic.keys()):
                        log.info("No par for {}".format(f))
                        continue
                    else:
                        par = par_dic[f]

                #Add job to pool
                job = pool.apply_async(allprocedures,
                                       (f, par, ),
                                       dict(emin=emin,
                                            emax=emax,
                                            trumpet=trumpet,
                                            keith=keith,
                                            clobber=clobber))
                jobs.append(job)

        #Run all jobs in parallel
        for job in jobs:
            job.get()

#niextract merge event file
#count rate cut applied in this step
def run_niextract(sourcename, max_date=None, output=None,
                  cut=2.0, filterbinsize=16):

    log.info('Running niextract-events')

    source_dir = os.getcwd().split('/')[-1]
    if sourcename != source_dir:
        check = pipeline_utils.outdircheck(sourcename)
        if check == -1:
            return

    #Build list of files to merge
    filenames = []
    invalid_columns = []
    for f in os.listdir("./"):
        if os.path.isfile(f+"/cleanfilt.evt"):
            tab = Table.read(f+"/cleanfilt.evt", hdu=1)
            date = datetime.datetime.strptime(
                    tab.meta['DATE-OBS'], '%Y-%m-%dT%H:%M:%S')
            #Date check for merge
            if max_date is not None:
                if date > max_date:
                    print("Excluding "+f+" from "+str(date))
                    continue
            #Some evts are missing columns...remember to look into this later
            if len(tab.columns) == 15:
                filenames.append(f+"/cleanfilt.evt")
            else:
                invalid_columns.append(f)

    #Write event file list
    with open(sourcename+"_evtlist", 'w') as f1:
        for fn in filenames:
            f1.write(fn+"\n")

    #Default output
    if output is None:
        output = sourcename+"_combined.evt"

    #Call niextract-events to merge files
    cmd = ['niextract-events', 'filename=@{}_evtlist'.format(sourcename),
           'eventsout={}'.format(output), 'clobber=yes']
    subprocess.call(cmd)

    #Save invalid column info
    with open('invalid_columns.txt', 'w') as f2:
        for fn in invalid_columns:
            f2.write(fn+"\n")

    #Run count rate cut
    if cut is not None:
        run_cr_cut(output, cut, filterbinsize)

#Perform count rate cut
def run_cr_cut(merged_evt, cut, filterbinsize):
	log.info("Running cr_cut cut={0} filterbinsize={1}".format(
                    str(cut), str(filterbinsize)))

	cmd = ['cr_cut.py',
		   '--cut', str(cut),
		   '--filterbinsize', str(filterbinsize),
		   merged_evt]

	subprocess.call(cmd)

#Update directory to download/pipe new data
def update(sourcename, heasarc_user, heasarc_pwd, decryptkey,
		   par, emin=0.25, emax=12, cut=2, filterbinsize=16,
		   trumpet=True, keith=True, crab=False, silent_curl=False):

    #Check the directory is correct
    source_dir = os.getcwd().split('/')[-1]
    log.info("Checking sourcename")
    if sourcename != source_dir:
        check = pipeline_utils.outdircheck(sourcename)
        if not check:
            return

    #Download new data
    pipeline_utils.run_datadownload(sourcename, heasarc_user,
            heasarc_pwd, './',
            decryptkey, clobber=False,
            silent_curl=silent_curl)

    #Launch multiprocessing wrapper for pip
    wrapper(par, emin=emin, emax=emax, trumpet=trumpet, keith=keith,
            clobber=False, crab=crab)

    #Merge and perform count rate cut
    run_niextract(sourcename, cut=cut, filterbinsize=filterbinsize)

    #Create merged evt backup
    merged_evt = sourcename+"_combined_cut.evt"
    message='Created with pulsar_pipe.update'
    pipeline_utils.product_backup(merged_evt, backupdir='evt_backups',
                                  message=message)

if __name__ == '__main__':
    parser = argparse.ArgumentParser(description=desc)

    #Arguments for accessing data from NASA site
    parser.add_argument("--user", help="heasarc username",
                        default=None, type=str)
    parser.add_argument("--passwd", help="heasarc password",
                        default=None, type=str)
    parser.add_argument("--outdir", help="output dir for download",
                        default="./", type=str)
    parser.add_argument("-k", help="decryption key", dest='key',
                        action=pipeline_utils.PasswordPromptAction,
                        type=str, required=False)
    parser.add_argument("--k_file", help="decryption key from file",
                        type=str, default=None)

    #Arguments for observation filtering
    parser.add_argument("--emin", help="Minimum energy to include",
                        type=float, default=0.25)
    parser.add_argument("--emax", help="Maximum energy to include",
                        type=float, default=12.0)
    parser.add_argument("--par", help="Par file to use for phases",
                        type=str, default="")
    parser.add_argument("--cut", help="Count rate for cut in cts/sec",
                        default=2.0, type=float)
    parser.add_argument("--filterbinsize", help="Bin size in seconds",
                        default=16.0, type=float)
    parser.add_argument("--trumpet", help="Run nicerl2 with trumpet cut",
                        default=True, type=bool)
    parser.add_argument("--keith",
                        help="Use standard filters from Keith Gendreau",
                        default=True, type=bool)
    parser.add_argument("--clobber", help="overwrite download files",
                        default=False, action='store_true')

    #Arguments for different function calls
    parser.add_argument("--update", default=None, type=str,
                        help="Update single source with full procedure")
    parser.add_argument("--obsID", help="ObsID for single psrpipe call",
                        default=None, type=str)
    parser.add_argument("--download", help="Run data download for source",
                        default=None, type=str)
    parser.add_argument("--backup", help="Create EVT backup for file",
                        default=None, type=str)
    parser.add_argument("--message", help="Log message for evt backup",
                        default="", type=str)
    #Two additional arguments for merge call
    parser.add_argument("--merge", help="combine evt with niextract-events",
                        default=None, type=str)
    parser.add_argument("--max_date", help="Max date to merge",
                        default=None, type=str)
    parser.add_argument("--output", help="Output name for merge",
                        default=None, type=str)

    #Option to silence curl progress bar
    parser.add_argument("--silent_curl", help="no curl progress bar",
                        default=False, action='store_true')

    #option to use crab multiprocessing setup
    parser.add_argument("--crab", help="run crab multiprocessing setup",
                        default=False, action='store_true')

    args = parser.parse_args()

    #Key handling
    if args.k_file is not None:
        with open(args.k_file, 'r') as f:
            args.key = f.readlines()[0].strip("\n")

    #Update directory and pipe all new data
    if args.update is not None:
        update(args.update, args.user, args.passwd,
               args.key, args.par,
               emin=args.emin, emax=args.emax,
               cut=args.cut,
               filterbinsize=args.filterbinsize,
               trumpet=args.trumpet,
               keith=args.keith,
               silent_curl=args.silent_curl)

    #Option to call pipeline on a single observation
    elif (args.obsID is not None) and (args.download is None):
        allprocedures(args.obsID, args.par,
                      args.emin, args.emax,
                      trumpet=args.trumpet,
                      keith=args.keith,
                      clobber=args.clobber,
                      crab=args.crab)
        if os.path.isfile(os.path.join(args.obsID+'_pipe', 'cleanfilt.evt')):
            check = input("Would you like to run count rate cut [y/n] -- ")
            if check in ['y', 'Y']:
                run_cr_cut(os.path.join(args.obsID+'_pipe', 'cleanfilt.evt'),
                                        args.cut, args.filterbinsize)

    #Option to only download the data with no filtering
    elif args.download is not None:
        assert(all([arg is not None for arg in [
                    args.user, args.passwd, args.key]]))
        pipeline_utils.run_datadownload(args.download,
                                        args.user, args.passwd,
                                        args.outdir,
                                        args.key, clobber=args.clobber,
                                        obsIDs=args.obsID,
                                        silent_curl=args.silent_curl)
    #Option to just merge
    #Optional arguments to merge up to specific date and give output name
    elif args.merge is not None:
        if args.max_date is None:
            run_niextract(args.merge,
                          cut=args.cut, filterbinsize=args.filterbinsize,
                          output=args.output)
        else:
            dt = datetime.datetime.strptime(args.max_date, '%Y-%m-%d')
            run_niextract(args.merge, max_date=dt,
                          cut=args.cut, filterbinsize=args.filterbinsize,
                          output=args.output)

    #Back up event file
    elif args.backup is not None:
        pipeline_utils.product_backup(args.backup, backupdir='evt_backups',
                                      message=args.message)

    else:
        log.warning("No method selected! Try 'pulsar_pipe.py --help'")


"""Implementing the tac shell command in python."""

"""We are importing sys module"""

import sys

"""we are importing only the required files i.e tac and readfile from a different file in same directory i.e lib.helper in our case."""

from lib.helper import tac, readfile

"""Initialising the TEXT variable with null value"""

TEXT = None

"""Initializing a variable ARG_CNT with a length function which calculates the length of the argument given in command line and reduces it by 1"""

ARG_CNT = len(sys.argv) - 1

"""condition : if value of ARG_CNT == 0 a TEXT variable is initialised with a input function using sys library and reads the file"""

if ARG_CNT == 0:
    TEXT = sys.stdin.read()

"""condition : if value of ARG_CNT == 1 a variable named filename is initialised and the filename is stored in it, then a new variable named TEXT is used to store a function call operation to return value"""

if ARG_CNT == 1:
    filename = sys.argv[1]
    TEXT = readfile(filename)

"""condition : if value of ARG_CNT is greater than 1, then it exectues the print statement."""

if ARG_CNT > 1:
    print(sys.argv[0], "doesn't handle more than one argument")

"""the print statement here calls a function tac which takes a TEXT variable as an argument"""

print(tac(TEXT))