import numpy as np
import tensorflow as tf
import cv2
import time

class_names = ['With mask', 'Without mask']

def detect_faces_and_classify(img, face_cascade, model):
    global class_names
    # obtenemos las caras usando el detector aplicado a la imagen
    faces = face_cascade.detectMultiScale(img, 1.1, 4)
    gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)

    for i, (x, y, w, h) in enumerate(faces):
        # sacamos la region de interes
        roi = gray[y:y + h, x:x + w]
        # resize para ingresar al modelo
        resized = cv2.resize(roi, (180, 180))
        # normalized = resized/255.
        reshaped = np.reshape(resized, (1, 180, 180, 1))

        # obtenemos las probabilidades para cada clase
        predictions = model.predict(reshaped)
        score = tf.nn.softmax(predictions[0])
        # generamos la label partiendo de las probabilidades
        label = np.argmax(score)

        # mostramos el recuadro con label rojo o verde dependiendo de lo obtenido
        # Red color for no mask
        color = (0, 0, 255)
        if label == 0:
            # Green color for mask
            color = (0, 255, 0)

        # agregamos la label y el rectangulo a la imagen
        cv2.putText(img, class_names[label], (x, y - 5), cv2.FONT_HERSHEY_SIMPLEX, 0.75, color, 2, cv2.LINE_AA)
        cv2.rectangle(img, (x, y), (x + w, y + h), color, 2)

# para probar la clasificacion en imagenes
def face_classifier():
    # tomamos el modelo guardado (proveniente del collab)
    model_path = "./saved_model/classifier"
    # cargamos el modelo
    model = tf.keras.models.load_model(model_path)
    # elegimos el detector de caras
    face_cascade = cv2.CascadeClassifier('haarcascade_frontalface_default.xml')

    # leemos las imagenes
    img = cv2.imread('./with_mask/1.png')
    img2 = cv2.imread('./without_mask/1.png')

    # le pasamos al metodo la imagen, el detector de caras y el modelo. Obtenemos la misma imagen con tags
    detect_faces_and_classify(img, face_cascade, model)
    detect_faces_and_classify(img2, face_cascade, model)

    # mostramos las imagenes
    cv2.imshow('With Mask', img)
    cv2.imshow('Without Mask', img2)
    cv2.waitKey()

def video_capture_classifier():
    # tomamos el modelo guardado (proveniente del collab)
    model_path = "./saved_model/classifier"
    # cargamos el modelo
    model = tf.keras.models.load_model(model_path)
    # elegimos el detector de caras
    face_cascade = cv2.CascadeClassifier('haarcascade_frontalface_default.xml')

    # capturamos imagenes de la webcam
    cap = cv2.VideoCapture(0)

    while True:
        _, frame = cap.read()
        flipped = cv2.flip(frame, 1)
        # a medida que lee cada frame, le pasamos el frame a la funcion. Obtenemos frame con tags
        detect_faces_and_classify(flipped, face_cascade, model)

        # mostramos el frame
        cv2.namedWindow('MaskNet', cv2.WINDOW_NORMAL)
        cv2.imshow('MaskNet', flipped)
        cv2.resizeWindow('MaskNet', 1000, 750)
        if cv2.waitKey(1) & 0xFF == ord('q'):
            break

def video_classifier():
    # tomamos el modelo guardado (proveniente del collab)
    model_path = "./saved_model/classifier"
    # cargamos el modelo
    model = tf.keras.models.load_model(model_path)
    # elegimos el detector de caras
    face_cascade = cv2.CascadeClassifier('haarcascade_frontalface_default.xml')

    # elegimos el path del video que queremos seleccionar
    video_path = "diego2rapido.mp4"
    # video_path = "florblancorapido.mp4"

    cap = cv2.VideoCapture(video_path)

    while cap.isOpened():
        ret, frame = cap.read()

        if not ret:
            print("Can't receive frame (stream end?). Exiting ...")
            break

        flipped = cv2.flip(frame, 1)
        # por cada frame del video aplicamos la funcion, obtenemos el frame con tags
        detect_faces_and_classify(flipped, face_cascade, model)

        # mostramos el frame
        cv2.namedWindow('MaskNet', cv2.WINDOW_NORMAL)
        cv2.imshow('MaskNet', flipped)
        cv2.resizeWindow('MaskNet', 1000, 750)
        if cv2.waitKey(1) & 0xFF == ord("q"):
            break

    cap.release()
    cv2.destroyAllWindows()

if __name__ == '__main__':
    # face_classifier()
    # video_capture_classifier()
    video_classifier()


import subprocess
import shutil
from pathlib import Path

import numpy as np
import tensorflow as tf
import ruamel.yaml

from model.models import AutoregressiveTransformer
from utils.scheduling import piecewise_linear_schedule, reduction_schedule

class ConfigManager:
  def __init__(self, config_path: str, model_kind: str, session_name: str = None):
    if model_kind not in ['autoregressive']:
      raise TypeError(f"model kind must be in {['autoregressive']}")
    self.config_path = Path(config_path)
    self.model_kind = model_kind
    self.yaml = ruamel.yaml.YAML()
    self.config, self.data_config, self.model_config = self._load_config()
    self.git_hash = self._get_git_hash()
    if session_name is None:
      if self.config['session_name'] is None:
        session_name = self.git_hash
    self.session_name = '_'.join(filter(None, [self.config_path.name, session_name]))
    self.base_dir, self.log_dir, self.train_datadir, self.weights_dir = self._make_folder_paths()
    self.learning_rate = np.array(self.config['learning_rate_schedule'])[0,1].astype(np.float32)  # set this up
    if model_kind == 'autoregressive':
      self.max_r = np.array(self.config['reduction_factor_schedule'])[0,1].astype(np.int32)  # set this up
      self.stop_scaling = self.config.get('stop_loss_scaling', 1.)  # set this up

  def _load_config(self):
    with open(str(self.config_path / 'data_config.yaml'), 'rb') as data_yaml:
      data_config = self.yaml.load(data_yaml)
    with open(str(self.config_path / f'{self.model_kind}_config.yaml'), 'rb') as model_yaml:
      model_config = self.yaml.load(model_yaml)
    all_config = {}
    all_config.update(model_config)
    all_config.update(data_config)
    return all_config, data_config, model_config

  @staticmethod
  def _get_git_hash():
    try:
      return subprocess.check_output(['git', 'describe', '--always']).strip().decode()
    except Exception as e:
      print(f'WARNING: could not retrieve git hash. {e}')

  def _check_hash(self):
    try:
      git_hash = subprocess.check_output(['git', 'describe', '--always']).strip().decode()
      if self.config['git_hash'] != git_hash:
        print(f"WARNING: git hash mismatch. Current: {git_hash}. Config hash: {self.config['git_hash']}")
    except Exception as e:
      print(f'WARNING: could not check git hash. {e}')

  def _make_folder_paths(self):
    base_dir = Path(self.config['log_directory']) / self.session_name
    log_dir = base_dir / f'{self.model_kind}_logs'
    weights_dir = base_dir / f'{self.model_kind}_weights'
    train_datadir = self.config['train_data_directory']   # set this up
    if train_datadir is None:
      train_datadir = self.config['data_directory']  # remember
    train_datadir = Path(train_datadir)
    return base_dir, log_dir, train_datadir, weights_dir

  @staticmethod
  def _print_dict_values(values, key_name, level= 0, tab_size = 2):
    tab = level * tab_size * ' '
    print(tab + '-', key_name, ':', values)

  def _print_dictionary(self, dictionary, recursion_level = 0):
    for key in dictionary.keys():
      if isinstance(key, dict):
        recursion_level += 1
        self._print_dictionary(dictionary[key], recursion_level)
      else:
        self._print_dict_values(dictionary[key], key_name = key, level = recursion_level)

  def print_config(self):
    print('\nCONFIGURATION', self.session_name)
    self._print_dictionary(self.config)

  def update_config(self):
    self.config['git_hash'] = self.git_hash
    self.model_config['git_hash'] = self.git_hash
    self.data_config['session_name'] = self.session_name
    self.model_config['session_name'] = self.session_name
    self.config['session_name'] = self.session_name

  def get_model(self, ignore_hash= False):
    if not ignore_hash:
      self._check_hash()
    if self.model_kind == 'autoregressive':
        #print(self.config['train_data_dir']y
        return AutoregressiveTransformer(encoder_model_dimension = self.config['encoder_model_dimension'],
                                       decoder_model_dimension = self.config['decoder_model_dimension'],
                                       encoder_num_heads = self.config['encoder_num_heads'],
                                       decoder_num_heads = self.config['decoder_num_heads'],
                                       encoder_num_layers = self.config['encoder_num_layers'],
                                       decoder_num_layers = self.config['decoder_num_layers'],
                                       encoder_maximum_position_encoding = self.config['encoder_maximum_position_encoding'],
                                       decoder_maximum_position_encoding = self.config['decoder_maximum_position_encoding'],
                                       encoder_feed_forward_dimension = self.config['encoder_feed_forward_dimension'],
                                       decoder_feed_forward_dimension = self.config['decoder_feed_forward_dimension'],
                                       dropout_rate = self.config['dropout_rate'],
                                       mel_start_value = self.config['mel_start_value'],
                                       mel_end_value = self.config['mel_end_value'],
                                       mel_channels = self.config['mel_channels'],
                                       encoder_vocab_size = self.config['encoder_vocab_size'],
                                       speaker_vocab_size = self.config['speaker_vocab_size'],
                                       speaker_feed_forward_dimension = self.config['speaker_feed_forward_dimension'],
                                       speaker_embedding_dimension = self.config['speaker_embedding_dimension'],
                                       decoder_prenet_dimensions = self.config['decoder_prenet_dimensions'],
                                       decoder_prenet_dropout = self.config['decoder_prenet_dropout'],
                                       stop_linear_dimension = self.config['stop_linear_dimension'],
                                       max_r = self.max_r, ###observe this ###
                                       phoneme_language= self.config['phoneme_language'],
                                       with_stress= self.config['with_stress'],
                                       debug = self.config['debug'],
                                       token_id_path = self.config['train_data_directory'],
                                       diagonal_bandwidth_b = self.config['diagonal_bandwidth_b'],
                                       diagonal_rate_regul_coeff = self.config['diagonal_rate_regul_coeff'],
                                       loss_weights = self.config['loss_weights'],
                                       layernorm = self.config['layernorm'],
                                       Ldc = self.config['Ldc'],
                                       attention_window = self.config['attention_window'],
                                       regul_coeff = self.config['diagonal_rate_regul_coeff'],
                                       diag_bandwidth = self.config['diagonal_bandwidth_b']
                                      )

  def compile_model(self, model):
    if self.model_kind == 'autoregressive':
      model._compile(stop_scaling = self.stop_scaling, optimizer = self.new_adam(self.learning_rate))

  @staticmethod
  def new_adam(learning_rate):
    return tf.keras.optimizers.Adam(learning_rate,
                                    beta_1= 0.9,
                                    beta_2= 0.98,
                                    epsilon= 1e-9)

  def dump_config(self):
    self.update_config()
    with open(self.base_dir / f'{self.model_kind}_config_yaml', 'w') as model_yaml:
      self.yaml.dump(self.model_config, model_yaml)
    with open(self.base_dir / 'data_config.yaml', 'w') as data_yaml:
      self.yaml.dump(self.data_config, data_yaml)

  def create_remove_dirs(self, clear_dir: False, clear_logs: False, clear_weights: False):
    self.base_dir.mkdir(exist_ok = True)
    if clear_dir:
      delete = input(f'Delete {self.log_dir} AND {self.weights_dir}? (y/[n])')
      if delete == 'y':
        shutil.rmtree(self.log_dir, ignore_errors= True)
        shutil.rmtree(self.weights_dir, ignore_errors= True)
    if clear_logs:
      delete = input(f'Delete {self.log_dir}? (y/[n])')
      if delete == 'y':
        shutil.rmtree(self.weights_dir, ignore_errors= True)
    if clear_weights:
      delete = input(f'Delete {self.weights_dir}? (y/[n])')
      if delete == 'y':
        shutil.rmtree(self.weights_dir, ignore_errors=True)
    self.log_dir.mkdir(exist_ok=True)
    self.weights_dir.mkdir(exist_ok=True)

  def load_model(self, checkpoint_path: str = None, verbose= True):
    model = self.get_model()
    self.compile_model(model)
    ckpt = tf.train.Checkpoint(net= model)
    manager = tf.train.CheckpointManager(ckpt, self.weights_dir,
                                        max_to_keep= None)
    if checkpoint_path:
      ckpt.restore(checkpoint_path)
      if verbose:
        print(f'restored weights from {checkpoint_path} at step {model.step}')
    else:
      if manager.latest_checkpoint is None:
        print(f'WARNING: could not find weights file. Trying to load from \n {self.weigths_dir}.')
        print('Edit data_config.yaml to point at the right log directory.')
        ckpt.restore(manager.latest_checkpoint)
      if verbose:
        print(f'restored weights from {manager.latest_checkpoint} at step {model.step}')

    #decoder_prenet_dropout = piecewise_linear_schedule(model.step, self.config['decoder_prenet_dropout_schedule']) # check this
    reduction_factor = None
    if self.model_kind == 'autoregressive': #### CHECK THIS ALSO ####
      reduction_factor = reduction_schedule(model.step, self.config['reduction_factor_schedule'])
    #model.set_constants(reduction_factor=reduction_factor, decoder_prenet_dropout=decoder_prenet_dropout) ### check this ###
    return model


from __future__ import print_function
import requests
import argparse
import json
from csiot import util

def enroll_node_cert(addr, cn):
	host, port = addr
	res = requests.get("http://%s:%s/enroll/cert?common_name=%s" % (host, port, cn))
	if res.status_code != requests.codes.ok:
		print("A server-side error occurred while enrolling certificate")
		return

	data = json.loads(res.content.decode("UTF-8"))

	util.store_cert("server", data["pem"])
	util.store_private_key("server", data["key"])
	util.store_cacert(data["ca"])

def enroll_node(addr, common_name):
	host, port = addr
	res = requests.get("http://%s:%s/enroll/node" % (host, port))
	if res.status_code != requests.codes.ok:
		print("A server-side error occurred while enrolling for a node-key")
		return

	data = json.loads(res.content.decode("UTF-8"))
	util.store_key("node", "node", data["node-key"])

	enroll_node_cert(addr, common_name)

def enroll_service(addr):
	host, port = addr
	res = requests.get("http://%s:%s/enroll/service" % (host, port))
	if res.status_code != requests.codes.ok:
		print("A server-side error occurred while enrolling for a service-key")
		return

	data = json.loads(res.content.decode("UTF-8"))
	util.store_key("node", "service", data["service-key"])

if __name__ == "__main__":
	parser = argparse.ArgumentParser()
	# node and service should take hostname:port pairs of their counterparts
	# e.g. python enroll.py --service node.domain.net:6667
	#      python enroll.py --node master.domain.net:6666
	parser.add_argument("--node", dest='node', required=False)
	parser.add_argument("--service", dest='service', required=False)
	parser.add_argument("--common-name", dest="common_name", default="localhost")
	args = parser.parse_args()

	# Default port 5000
	if args.node:
		if ':' not in args.node:
			args.node += ":5000"

		enroll_node(args.node.split(":"), args.common_name)

	if args.service:
		enroll_service(args.node.split(":"))


from ellipticcurve.ecdsa import Ecdsa
from ellipticcurve.privateKey import PrivateKey

# Gerando as chaves
chave_privada = PrivateKey()
chave_publica = chave_privada.publicKey()

mensagem = "Segredo secreto :O"

# Gerando a assinatura
carloscabral = Ecdsa.sign(mensagem,chave_privada)

# Verificando
Resposta = Ecdsa.verify(mensagem,carloscabral,chave_publica)
print(Resposta)


from unittest import TestCase

from openslides_backend.shared.patterns import (
    Collection,
    FullQualifiedField,
    FullQualifiedId,
)

from .utils import Client, ResponseWrapper, create_test_application

class WSGIApplicationTester(TestCase):
    """
    Tests the WSGI application in general.
    """

    def setUp(self) -> None:
        self.application = create_test_application(
            user_id=0, view_name="ActionsView"
        )  # User is anonymous

    # def test_create_application(self) -> None:
    #     # This test does not use our test application but the real one.
    #     from openslides_backend.main import application
    #
    #     self.assertTrue(isinstance(application, OpenSlidesBackendESGIApplication))

    # def test_create_application_2(self) -> None:
    #     # This test does not use our test application but the real one.
    #     from openslides_backend.main import create_application
    #
    #     os.environ["OPENSLIDES_BACKEND_DEBUG"] = "1"
    #     app = create_application()
    #     self.assertTrue(isinstance(app, OpenSlidesBackendApplication))

    def test_wsgi_request_wrong_method(self) -> None:
        client = Client(self.application, ResponseWrapper)
        response = client.get("/")
        self.assertEqual(response.status_code, 405)

    def test_wsgi_request_wrong_media_type(self) -> None:
        client = Client(self.application, ResponseWrapper)
        response = client.post("/")
        self.assertEqual(response.status_code, 400)
        self.assertIn("Wrong media type.", str(response.data))

    def test_wsgi_request_missing_body(self) -> None:
        client = Client(self.application, ResponseWrapper)
        response = client.post("/", content_type="application/json")
        self.assertEqual(response.status_code, 400)
        self.assertIn("Failed to decode JSON object", str(response.data))

    def test_wsgi_request_fuzzy_body(self) -> None:
        client = Client(self.application, ResponseWrapper)
        response = client.post(
            "/", json={"fuzzy_key_Eeng7pha3a": "fuzzy_value_eez3Ko6quu"},
        )
        self.assertEqual(response.status_code, 400)
        self.assertIn("data must be array", str(response.data))

    def test_wsgi_request_fuzzy_body_2(self) -> None:
        client = Client(self.application, ResponseWrapper)
        response = client.post(
            "/", json=[{"fuzzy_key_Voh8in7aec": "fuzzy_value_phae3iew4W"}],
        )
        self.assertEqual(response.status_code, 400)
        self.assertIn(
            "data[0] must contain [\\'action\\', \\'data\\'] properties",
            str(response.data),
        )

    def test_wsgi_request_no_existing_action(self) -> None:
        client = Client(self.application, ResponseWrapper)
        response = client.post(
            "/", json=[{"action": "fuzzy_action_hamzaeNg4a", "data": [{}]}],
        )
        self.assertEqual(response.status_code, 400)
        self.assertIn(
            "Action fuzzy_action_hamzaeNg4a does not exist.", str(response.data)
        )

    def test_health_route(self) -> None:
        client = Client(self.application, ResponseWrapper)
        response = client.get("/health")
        self.assertEqual(response.status_code, 200)
        self.assertIn("healthinfo", str(response.data))

class TypesTester(TestCase):
    """
    Tests some utils and types
    """

    def test_collection(self) -> None:
        collection = Collection("collection_vi8cah2Eih")
        self.assertEqual(str(collection), "collection_vi8cah2Eih")

    def test_collection_repr(self) -> None:
        collection = Collection("collection_yag3Boaqui")
        self.assertEqual(repr(collection), "Collection collection_yag3Boaqui")

    def test_collection_comparing(self) -> None:
        collection_1 = Collection("collection_aeboo3ieSh")
        collection_2 = Collection("collection_aeboo3ieSh")
        self.assertEqual(collection_1, collection_2)

    def test_collection_hashing(self) -> None:
        collection = Collection("collection_Din9chosoo")
        self.assertEqual(hash(collection), hash("collection_Din9chosoo"))

    def test_full_qualified_id(self) -> None:
        fqid = FullQualifiedId(Collection("collection_Aid6ahdooT"), 8283937728)
        self.assertEqual(str(fqid), "collection_Aid6ahdooT/8283937728")

    def test_full_qualified_id_repr(self) -> None:
        fqid = FullQualifiedId(Collection("collection_oozaiX1pee"), 7099085886)
        self.assertEqual(repr(fqid), "FQId collection_oozaiX1pee/7099085886")

    def test_full_qualified_id_comparing(self) -> None:
        fqid_1 = FullQualifiedId(Collection("collection_reeyie3Woo"), 2133862900)
        fqid_2 = FullQualifiedId(Collection("collection_reeyie3Woo"), 2133862900)
        self.assertEqual(fqid_1, fqid_2)

    def test_full_qualified_id_hashing(self) -> None:
        fqid = FullQualifiedId(Collection("collection_ia5Ooyuiso"), 9638688299)
        self.assertEqual(hash(fqid), hash("collection_ia5Ooyuiso/9638688299"))

    def test_full_qualified_field(self) -> None:
        fqfield = FullQualifiedField(
            Collection("collection_Shoo1uut4u"), 7208641662, "field_ais1aBau6d"
        )
        self.assertEqual(
            str(fqfield), "collection_Shoo1uut4u/7208641662/field_ais1aBau6d"
        )

    def test_full_qualified_field_repr(self) -> None:
        fqfield = FullQualifiedField(
            Collection("collection_yaiS2uthi8"), 78718784720, "field_ueth3Rohv8"
        )
        self.assertEqual(
            repr(fqfield), "FQField collection_yaiS2uthi8/78718784720/field_ueth3Rohv8"
        )

    def test_full_qualified_field_comparing(self) -> None:
        fqfield_1 = FullQualifiedField(
            Collection("collection_ioMohcui0u"), 7208641662, "field_epee2jeRee"
        )
        fqfield_2 = FullQualifiedField(
            Collection("collection_ioMohcui0u"), 7208641662, "field_epee2jeRee"
        )
        self.assertEqual(fqfield_1, fqfield_2)

    def test_full_qualified_field_hashing(self) -> None:
        fqfield = FullQualifiedField(
            Collection("collection_ohf3Thoo9i"), 8432643375, "field_Raechee5ee"
        )
        self.assertEqual(
            hash(fqfield), hash("collection_ohf3Thoo9i/8432643375/field_Raechee5ee")
        )

    def test_full_qualified_field_fqid(self) -> None:
        fqfield = FullQualifiedField(
            Collection("collection_quephah8Oo"), 3148072663, "field_Ein2Aos0Ku"
        )
        self.assertEqual(
            fqfield.fqid,
            FullQualifiedId(Collection("collection_quephah8Oo"), 3148072663),
        )


# -*- coding: utf-8 -*-
#!/usr/bin/env python2
"""
API통합 모듈
해당 모듈을 통해 API를 사용
"""

from slackclient import SlackClient
from config import CrawlerConfig

class SlackAPI:
	"""
	Description
		네이버 메인뉴스 크롤링 데이터를 가공하는 클래스

	Func
		__init__ : 생성자. 초기 셋팅 작업
		process_item : 전체 프로세스를 담당하는 함수
		send_news_to_slack : slack에 뉴스를 발송하는 함수

	Attr
		None
	"""

	def __init__(self):
		self.sc = SlackClient(CrawlerConfig().bot_token)

	def call_send_api(self, msg):
		"""
		Description
			API를 호출하는 함수
			만약 다른 채널 혹은 사용자에게 메세지를 보내기 위해서는

		Args
			msg : 발송하려는 메세지가 담겨있는 변수
		"""

		self.sc.api_call(
		"chat.postMessage", channel="#every_news_playwings", text=msg,
		username='every_news', icon_emoji=':robot_face:'
		)


import json
import cv2

labels = {
            'left': 0,
            'right': 1,
            'front': 2,
            'behind': 3
        }

data_root = '/home/asaran/research/spatial-relations-attention/data/CLEVR_v1.0/'
with open(data_root+'scenes/CLEVR_train_scenes.json') as datafile:
    train_data = json.load(datafile)

with open(data_root+'scenes/CLEVR_val_scenes.json') as datafile:
    test_data = json.load(datafile)

train_data_list = []
test_data_list = []

for img in train_data['scenes']:
    objects = img['objects']
    #print('image_index: '+str(img['image_index']))
    #print('len(objects):' + str(len(objects)))
    #im = cv2.imread(data_root+'images/'+img['image_filename'])
    #width, height =
    for relation in img['relationships']:
        for subj in range(len(img['relationships'][relation])):
            for obj in img['relationships'][relation][subj]:
                #print('obj: '+str(obj))
                #print('subj: '+str(subj))
                data_dict = {}
                data_dict['img_path'] = data_root+'images/train/'+img['image_filename']
                obj_pos = objects[obj]['pixel_coords']
                subj_pos = objects[subj]['pixel_coords']

                pos = obj_pos + subj_pos
                data_dict['position'] = pos
                data_dict['label'] = labels[relation]

                train_data_list.append(data_dict)

train_outfile = '/media/internal/asaran/CLEVR/train_relations.json'
with open(train_outfile, 'w') as outfile:
    json.dump(train_data_list, outfile)

for img in test_data['scenes']:
    objects = img['objects']
    #im = cv2.imread(data_root+'images/'+img['image_filename'])
    #width, height =
    for relation in img['relationships']:
        for subj in range(len(img['relationships'][relation])):
            for obj in img['relationships'][relation][subj]:
                data_dict = {}
                data_dict['img_path'] = data_root+'images/val/'+img['image_filename']
                obj_pos = objects[obj]['pixel_coords']
                subj_pos = objects[subj]['pixel_coords']

                pos = obj_pos + subj_pos
                data_dict['position'] = pos
                data_dict['label'] = labels[relation]

                test_data_list.append(data_dict)

test_outfile = '/media/internal/asaran/CLEVR/test_relations.json'
with open(test_outfile, 'w') as outfile:
    json.dump(test_data_list, outfile)


#!/usr/bin/env python

import sys
import argparse
from github import Github

def parse_args(args):
        parser = argparse.ArgumentParser()
        parser.add_argument("--repo_list",
                            action="store")
        parser.add_argument("--pull_id_list",
                            action="store")
        parser.add_argument("--repo",
                            action="store")
        parser.add_argument("--pull_id",
                            action="store")
        parser.add_argument("--jenkins_url",
                            action="store")
        parser.add_argument("--status",
                            action="store")
        parser.add_argument("--description",
                            action="store")
        parser.add_argument("--home",
                            action="store")
        args = parser.parse_args(args)
        return args

args = parse_args(sys.argv[1:])
repo_list=[]
pull_id_list=[]
if args.pull_id_list != "none":
    tmp_repo_list = args.repo_list.split(',')
    for n, pull_id in enumerate(args.pull_id_list.split(',')):
        if pull_id.isalnum():
            repo_list.append(tmp_repo_list[n])
            pull_id_list.append(long(pull_id))
    args.description += "[dependency]"
elif args.repo != "none":
    repo = args.repo
    repo_list.append(repo)
    pull_id_list.append(long(args.pull_id))

with open('{0}/.ghtoken'.format(args.home), 'r') as file:
    TOKEN = file.read().strip('\n')
gh = Github(TOKEN)

for n, repo in enumerate(repo_list):
    pr = gh.get_repo(repo).get_pull(long(pull_id_list[n]))
    COMMIT = pr.get_commits().reversed[0]
    COMMIT.create_status(args.status, \
                        description=args.description.replace(',',' '), \
                        context='Jenkins')


"""
Wide ResNet by Sergey Zagoruyko and Nikos Komodakis
Fixup initialization by Hongyi Zhang, Yann N. Dauphin, Tengyu Ma
Based on code by xternalz and Andy Brock:
https://github.com/xternalz/WideResNet-pytorch
https://github.com/ajbrock/BoilerPlate
"""

import math
import torch
import torch.nn as nn
import torch.nn.functional as F

def get_Q(t,dims=[0]):
    a = t - torch.mean(t,[0])
    return a*a

class Swish(nn.Module):

    def forward(self, input):
        return (input + torch.sigmoid(input))

    def __repr__(self):
        return self.__class__.__name__ + '  ()'

class BasicBlock(nn.Module):
    droprate = 0.0
    use_bn = True
    use_fixup = False
    fixup_l = 12
    varnet = False
    sigmaW = 1.0
    def __init__(self, in_planes, out_planes, stride):
        super(BasicBlock, self).__init__()

        self.bn = nn.BatchNorm2d(in_planes)

##        self.bn = nn.BatchNorm2d(in_planes)
##        self.relu = nn.Softplus()
##        self.relu = Swish()
        self.relu = nn.ReLU(inplace=True)
##        self.relu = nn.Hardtanh()
        print(self.relu)

        gain = torch.nn.init.calculate_gain('relu')
        self.conv = nn.Conv2d(
            in_planes, out_planes, kernel_size=3, stride=stride, padding=1, bias=False
        )

        self.equalInOut = in_planes == out_planes

        self.conv_res = nn.Conv2d(
            in_planes, out_planes, kernel_size=1, stride=stride, padding=0, bias=False
        )
        self.conv_res = not self.equalInOut and self.conv_res or None
        ##self.no_act =  not self.equalInOut

        ##print("Using linear networks!")

        assert (
            self.use_fixup or self.use_bn
        ), "Need to use at least one thing: Fixup or BatchNorm"

        self.scale = nn.Parameter(torch.ones(1))
        self.biases = nn.ParameterList(
            [nn.Parameter(torch.zeros(1)) for _ in range(2)]
        )

        k = (
            self.conv.kernel_size[0]
            * self.conv.kernel_size[1]
            * self.conv.out_channels
        )

        if self.varnet:
            self.conv.weight.data.normal_(
                0, gain * math.sqrt(self.sigmaW / k)
            )
        else:
            if self.sigmaW == 0.0:
                self.conv.weight.data.zero_()
            else:
                self.conv.weight.data.normal_(
                    0, gain * math.sqrt(self.sigmaW / k)
                )

        if self.conv_res is not None:
            gain = 1.0 ##doesn't currently pass through relu.
            k = (
                self.conv_res.kernel_size[0]
                * self.conv_res.kernel_size[1]
                * self.conv_res.out_channels
            )
            if self.varnet:
                self.conv_res.weight.data.normal_(0, gain * math.sqrt(self.sigmaW / k))
            else:
                ConstAvg(self.conv_res.weight, self.conv_res.bias)
                ##self.conv_res.weight.data.fill_(gain**2 / self.conv_res.in_channels)

    def forward(self, x):
        if self.use_bn:
            x_out = self.bn(x)

            ##x_out = x
            ##out = x_out
            ##out = self.relu(self.conv(x_out + self.biases[0]))

            out = self.conv(self.relu(x_out))
            ##out = self.conv(x_out)
            if self.droprate > 0:
                out = F.dropout(out, p=self.droprate, training=self.training)
        else:
            x_out = x + self.biases[0]
            out = self.conv(self.relu(x_out)) + self.biases[1]
            ##out = self.noise(out)

##            x_out = self.relu(x + self.biases[0])
##            out = self.conv(x_out) + self.biases[1]

            if self.droprate > 0:
                out = F.dropout(out, p=self.droprate, training=self.training)

            out = self.scale * out

        if self.equalInOut:
            out =  torch.add(x, out)
        else:
            out = torch.add(self.conv_res(x_out), out)

        return out

class NetworkBlock(nn.Module):
    def __init__(self, nb_layers, in_planes, out_planes, block, stride):
        super(NetworkBlock, self).__init__()
        self.layer = self._make_layer(block, in_planes, out_planes, nb_layers, stride)

    def _make_layer(self, block, in_planes, out_planes, nb_layers, stride):
        layers = []

        for i in range(int(nb_layers)):
            _in_planes = i == 0 and in_planes or out_planes
            _stride = i == 0 and stride or 1
            layers.append(block(_in_planes, out_planes, _stride))

        return nn.Sequential(*layers)

    def forward(self, x):

        out = self.layer(x)

        return self.layer(x)

class WideResNet(nn.Module):
    def __init__(
        self,
        depth,
        num_classes,
        widen_factor=1,
        droprate=0.0,
        use_bn=True,
        use_fixup=False,
        varnet=False,
        noise=0.0,
        lrelu=0.0,
        sigmaW = 1.0,
        init = None,
        dropl1 = False,
    ):
        super(WideResNet, self).__init__()

        nChannels = [16, 16 * widen_factor, 32 * widen_factor, 64 * widen_factor]

        assert (depth - 4) % 3 == 0, "You need to change the number of layers"
        n = (depth - 4) / 3

        BasicBlock.droprate = droprate
        BasicBlock.use_bn = use_bn
        BasicBlock.fixup_l = n * 3
        BasicBlock.use_fixup = use_fixup
        BasicBlock.varnet = varnet
        BasicBlock.sigmaW = sigmaW

        self.droprate = droprate
        self.dropl1 = dropl1
        ##print("Use fixup WideResnet:")
        ##print(use_fixup)
        ##print("Use BN WideResnet:")
        ##print(use_bn)
        block = BasicBlock

        self.conv1 = nn.Conv2d(
            3, nChannels[0], kernel_size=3, stride=1, padding=1, bias=False
        )

        k = (
            self.conv1.kernel_size[0]
            * self.conv1.kernel_size[1]
            * self.conv1.out_channels
        )

        gain = torch.nn.init.calculate_gain('relu')
        if varnet:
            self.conv1.weight.data.normal_(0, 1.0 * math.sqrt(sigmaW / k))
        else:
            ##makeLambdaDeltaOrthogonal(self.conv1.weight, self.conv1.bias, 1/3)
            ConstAvg(self.conv1.weight, self.conv1.bias)

        self.block1 = NetworkBlock(n, nChannels[0], nChannels[1], block, 1)
        self.block2 = NetworkBlock(n, nChannels[1], nChannels[2], block, 2)
        self.block3 = NetworkBlock(n, nChannels[2], nChannels[3], block, 2)

        self.relu = nn.ReLU(inplace=True)

##        self.relu = nn.Softplus()
##        self.relu = Swish()

        self.fc = nn.Linear(nChannels[3], num_classes)
        self.nChannels = nChannels[3]

        self.fc.bias.data.zero_()
        self.fc.weight.data.zero_()

        for m in self.modules():
            if isinstance(m, nn.BatchNorm2d):
                m.weight.data.fill_(1)
                m.bias.data.zero_()

    def forward(self, x):
        out = self.conv1(x)
        if self.dropl1:
            out = F.dropout(out, p=self.droprate, training=self.training)

        out = self.block1(out)
        out = self.block2(out)
        out = self.block3(out)
        out = self.relu(out)
        out = F.avg_pool2d(out, 8)
        out = out.view(-1, self.nChannels)
        return self.fc(out)

def genOrthgonal(dim):
    a = torch.zeros((dim, dim)).normal_(0, 1)
    q, r = torch.qr(a)
    d = torch.diag(r, 0).sign()
    diag_size = d.size(0)
    d_exp = d.view(1, diag_size).expand(diag_size, diag_size)
    q.mul_(d_exp)
    return q

def makeLambdaDeltaOrthogonal(weights, bias, gain):
    rows = weights.size(0)
    cols = weights.size(1)
    if bias is not None:
        nn.init.constant_(bias, 0)
    dim = max(rows, cols)
    mid1 = weights.size(2) // 2
    mid2 = weights.size(3) // 2
    nn.init.constant_(weights, 0)
    weights.data[:, :, mid1, mid2] = gain ##torch.ones([rows, cols]) * gain

def ConstAvg(weights, bias, gain=1.0, const = 1.0):
    rows = weights.size(0)
    cols = weights.size(1)

    if const < 1.0:
        k = cols * weights.size(2) * weights.size(3)
        weights.data.normal_(0, math.sqrt(gain*(1.0-const) / k))
    else:
        nn.init.constant_(weights, 0)

    if bias is not None:
        nn.init.constant_(bias, 0)

    if const > 0.0:

        mid1 = weights.size(2) // 2
        mid2 = weights.size(3) // 2
        factor = const/cols

        weights.data[:, :, mid1, mid2] += factor

def ConstIdentity(weights, bias, gain, const = 1.0):

    rows = weights.size(0)
    cols = weights.size(1)

    assert(rows==cols)

    if const < 1.0:
        k = cols * weights.size(2) * weights.size(3)
        weights.data.normal_(0, math.sqrt(gain*(1.0-const) / k))
    else:
        nn.init.constant_(weights, 0)

    if bias is not None:
        nn.init.constant_(bias, 0)

    if const > 0.0:
        mid1 = weights.size(2) // 2
        mid2 = weights.size(3) // 2
##        factor = (1.0/cols*rows)*const
        factor = const

        for d0 in range(rows):
            weights.data[d0, d0, mid1, mid2] += factor


# -*- coding: utf-8 -*-
"""
Created on Wed Oct 23 15:05:05 2019

@author: bguilfoyle
"""
import numpy as np
import astropy.io.fits as pyfits
import glob
import matplotlib
import matplotlib.animation as animation
import matplotlib.pyplot as plt
import matplotlib.colors as mcolors
from tkinter import *

#==============================================================================
def lingray(x, a=None, b=None):
    """
    Overview: Auxiliary function that specifies the linear gray scale.
    a and b are the cutoffs : if not specified, min and max are used
    """
    if a == None:
        a = np.min(x)
    if b == None:
        b = np.max(x)
    return 255.0 * (x-float(a))/(b-a)
#==============================================================================

#==============================================================================
def loggray(x, a=None, b=None):
    """
    Overview: Auxiliary function that specifies the logarithmic gray scale.
    a and b are the cutoffs : if not specified, min and max are used
    """
    if a == None:
        a = np.min(x)
    if b == None:
        b = np.max(x)
    linval = 10.0 + 990.0 * (x-float(a))/(b-a)
    return (np.log10(linval)-1.0)*0.5 * 255.0
#==============================================================================

#==============================================================================
def correctedImage(raw,dark,flat,bias):
    """
    Overview: Return a corrected image based off basic raw reduction
    """
    return (raw - bias - dark) / flat
#==============================================================================

def addPlot(image,colour,newAlpha):
    """
    Overview: Make a plot using plt.imshow
    """
    plt.imshow(loggray(image), cmap= colour, alpha = newAlpha)
    plt.colorbar()
#==============================================================================

path = "C:/Users/bguilfoyle/Documents/CompPhysics/FYP/seyfertGalaxyThesis/data/seyfertImages/"
rawRPath = path + "raw/*R.fit"
rawHPath = path + "raw/*H.fit"
rawSIIPath = path + "raw/*SII.fit"
rawVPath = path + "raw/*V.fit"
rawPath = path + "raw/*.fit"
biasPath = path + "bias/*.fit"
darkPath = path + "dark/*bin2.fit"
flatPath = path + "flat/Flat_bin2*.fit"

raw = glob.glob(rawRPath)
bias = glob.glob(biasPath)
dark = glob.glob(darkPath)
flat = glob.glob(flatPath)

avgRaw = avgImage(raw)
avgBias = avgImage(bias)
avgDark = avgImage(dark)
avgFlat = avgImage(flat)

finalCombined = correctedImage(avgRaw,avgDark,avgFlat,avgBias)
plt.imshow(loggray(finalCombined), cmap = "Reds")
plt.colorbar()


#!/usr/bin/env python3
# -*- coding: utf-8 -*-+

"""  """
import keras.backend as K

""" tensorflow """
import tensorflow as tf

"""  """
import numpy as np

""" lib """
from .actor_network import ActorNetwork
from .critic_network import CriticNetwork
from .replay_buffer import ReplayBuffer

import math

MAX_EPISODES = 1000
MAX_EP_STEPS = 200
GAMMA = 0.99

class DDPG(object):
    def __init__(self,env,state_size,action_size,action_bound):
        self.state_size = state_size
        self.action_size = action_size
        self.action_bound = action_bound
        self.env = env

        """ Tensorflow GPU optimization """
        config = tf.ConfigProto()
        config.gpu_options.allow_growth = True
        self.sess = tf.Session(config=config)
        K.set_session(self.sess)

        """ init """
        self.Init_Params()

        """ network """
        self.actor = ActorNetwork(self.sess,state_size,action_size,self.batch_size,self.TAU,self.lr_a)
        self.critic = CriticNetwork(self.sess,state_size,action_size,self.batch_size,self.TAU,self.lr_c)

        self.actor.Load_Model()
        self.critic.Load_Model()
        """ replayBuffer """
        self.replay_buffer = ReplayBuffer(self.buffer_size,1337)

    def Init_Params(self):
        """ Target Network HyperParameters """
        self.TAU = 0.001

        """ learning rate """
        self.lr_a = 0.0001
        self.lr_c = 0.001

        """ seed """
        np.random.seed(1337)

        """ buffer size """
        self.buffer_size = 100000

        """ batch size """
        self.batch_size = 32

    def Sim_Train_HER(self,train = 0):
        for i in range(MAX_EPISODES):
            state = self.env.reset(True)
            state = np.reshape(state,(1,self.state_size))

            ep_reward = 0
            ep_ave_max_q = 0
            test = 2.0
            for j in range(MAX_EP_STEPS):
            # for j in range(self.env.spec.timestep_limit):
                loss = 0
                self.env.render()

                action = self.actor.Evaluate_Actor(state)
                action_ = np.clip(np.random.normal(action,1), *self.env.action_bound)
                # print(action)
                # if(action[0][0] >= action[0][1]):
                #     action_ = 0
                # else:
                #     action_ = 1

                # tmp = abs(action[0])
                # action_ = np.argmax(tmp)
                # action_ = 0

                # new_state,reward,done,info = self.env.step(action)
                new_state,reward,done,info = self.env.step(action_)

                self.replay_buffer.Add_Buffer(np.reshape(state,(self.state_size,)),\
                                              np.reshape(action,(self.action_size,)),\
                                              reward,\
                                              np.reshape(new_state,(self.state_size,)),\
                                              done)

                """ update batch """
                s_batch,a_batch,r_batch,s2_batch,d_batch,y_t = self.replay_buffer.Get_Batch(self.batch_size)

                target_q_value = self.critic.Evaluate_Target_Actor([s2_batch,self.actor.Evaluate_Target_Actor(s2_batch)])

                for k in range(len(r_batch)):
                    if(d_batch[k]):
                        y_t[k] = r_batch[k]
                    else:
                        y_t[k] = r_batch[k] + GAMMA*target_q_value[k]
                if(train):
                    # print(y_t)
                    loss += self.critic.model.train_on_batch([s_batch,a_batch], y_t)
                    a_for_grad = self.actor.Evaluate_Actor(s_batch)
                    grads = self.critic.Gradient(s_batch,a_for_grad)
                    self.actor.Train(s_batch,grads)
                    self.actor.Target_Train()
                    self.critic.Target_Train()

                ep_reward += reward
                state = np.reshape(new_state,(1,self.state_size))
                # print("Episode", i, "Step", j, "Actions", action, "Reward", reward,'loss',loss)
                print("Episode", i, "Step", j,"Actions", action, 'action',action_,"Reward", reward,'loss',loss)

                if(done):
                    break
            print("TOTAL REWARD @ " + str(i) +"-th Episode  : Reward " + str(ep_reward))
            print("Total Step: " + str(j))
            print("")
        if(train):
            print("save model")
            self.actor.Save_Model()
            self.actor.Save_Weight_JSON()
            self.critic.Save_Model()
            self.critic.Save_Weight_JSON()
        print("Finish")


import pandas as pd
import numpy as np
from sklearn.metrics import classification_report, confusion_matrix
from sklearn.exceptions import UndefinedMetricWarning

import random as rd
import sys

import warnings
warnings.filterwarnings("ignore", category=DeprecationWarning)
warnings.filterwarnings("ignore", category=RuntimeWarning)
warnings.filterwarnings("ignore", category=UndefinedMetricWarning)

from config.config import *

from detection.detection import Detection
from detection.preprocessing.preprocessing import process_labels

MIN_SIZE = 1
MAX_SIZE = max(MIN_SIZE+1, 20)

np.set_printoptions(linewidth=150, precision=2)

def outlier_report(df_y, outlier_indices):
    """

    :param df_y:
    :param outlier_indices:

    :return:
    """
    detected_attack = 0
    detected_normal = 0
    false_negtive = 0
    false_positive = 0
    count = 0
    max_count = len(outlier_indices)

    for i in range(df_y.shape[0]):
        label_num = df_y[i]

        if count < max_count and i == outlier_indices[count]:
            if label_num != 0:
                detected_attack += 1
            else:
                false_positive += 1
            count += 1
        else:
            if label_num != 0:
                false_negtive += 1
            else:
                detected_normal += 1
    total = detected_attack + detected_normal + false_positive + false_negtive
    print("")
    print("=========================================")
    print("             OUTLIER ACCURACY            ")
    print("=========================================")
    print("          |   Det Attack       Det Normal  ")
    print("----------|--------------------------------")
    print("Is Attack |   ", detected_attack, "          ", false_negtive)
    print("Is Normal |   ", false_positive, "           ", detected_normal)
    print("")
    print("          |  Det Attack        Det Normal ")
    print("-----------------------------------------")
    print("Is Attack |  ", round(detected_attack / total * 100, 4), "         ", round(false_negtive / total * 100, 4))
    print("Is Normal |  ", round(false_positive / total * 100, 4), "          ",
          round(detected_normal / total * 100, 4))
    print("")
    if detected_attack > 0:
        precision = detected_attack/(detected_attack + false_positive)
        recall = detected_attack/(detected_attack + false_negtive)
        f1 = (2 * precision * recall) / (precision + recall)
    else:
        precision = 0
        recall = 0
        f1 = 0

    print("Precision: ", round(precision, 3))
    print("Recall: ", round(recall, 3))
    print("F1-Score: ", round(f1, 3))
    return [precision, recall, f1]

def classification_result(y, y_pred):
    """

    :param y:
    :param y_pred:

    :return:
    """
    assert len(y) == len(y_pred)
    correct = []
    wrong = []
    for i in range(len(y)):
        if y[i] == y_pred[i]:
            correct.append(i)
        else:
            wrong.append(i)
    return correct, wrong

def read_file(name):
    """

    :param name:

    :return:
    """
    df = pd.read_csv(name, na_values="Infinity", dtype=dtypes)
    df = df.replace("Infinity", sys.maxsize)
    df = df.fillna(sys.maxsize)

    df_X = df.iloc[:, 0:-1].copy()
    df_X = df_X.apply(pd.to_numeric)

    df_y = df.iloc[:, -1].copy()
    df_y = process_labels(df_y, labels, classes)

    #df_X = df_X.sample(frac=1)
    #df_y = df_y.iloc[df_X.index]

    return df_X, df_y

if __name__ == '__main__':

    data_directoy = "../TestData/"
    filemon = "normal.csv"
    filetue = "dos.csv"
    filewed = "portscan.csv"
    filethr1 = "ddos.csv"

    file_list = [filemon, filetue, filewed, filethr1]
    file_list = [data_directoy + x for x in file_list]

    print("Loading data...")
    df_X, df_y = read_file(file_list[0])
    new_features = df_X.columns

    det = Detection(classes=classes , threshold_percentile=threshold_percentile, truth_size=truth_size,
                    truth_update_frac=truth_update_frac, truth_save_folder=truth_save_folder,
                    outlier_save_folder=outlier_save_folder, classifier_save_folder=classifier_save_folder)

    print("Initializing detection system...")
    det.initialize(df_X, df_X, np.ndarray.flatten(df_y.values))
    print("Training Complete")

    name = ["DoS", "PortScan", "DDoS"]

    for i in range(1,len(file_list)):
        print(name[i-1] + " Test...")
        df_X, df_y = read_file(file_list[i])
        y_pred = []
        outlier_indices = []
        size = df_X.shape[0]
        c = 0
        index = df_X.index.tolist()
        for k in range(len(index)):
            j = index[k]
            if c % 50 == 0 and c > 0:
                print("\rStep " + str(c) +" of " + str(size), sep=' ', end='', flush=True)
                #print("Step " + str(c) +" of " + str(size))
                """
                y_now = df_y.iloc[index[:k]]
                cls_rep = classification_report(y_now.values, y_pred, labels=classes, output_dict=True)
                print("F1 Scores => ", end='')
                print("Normal: ", np.round(cls_rep['0']['f1-score'],2), end='')
                print(" | Dos: ", np.round(cls_rep['1']['f1-score'],2), end='')
                print(" | PortScan: ", np.round(cls_rep['2']['f1-score'],2), end='')
                print(" | DDoS: ", np.round(cls_rep['3']['f1-score'],2), end='')
                print(" | Weighted: ", np.round(cls_rep['weighted avg']['f1-score'],2))
                """
            c += 1

            X = df_X.iloc[[j]]
            y = df_y.iloc[[j]]
            outlier = det.detect_outliers(X)
            if outlier:
                outlier_indices.append(j)
                pred = det.classfiy(X)[0]
                if pred == 0:
                    det.update_outlier(X)
                det.update_classifier(X, np.ndarray.flatten(y.values))
                y_pred.append(pred)
            else:
                det.update_outlier(X)
                y_pred.append(0)
        print("")
        print(classification_report(df_y.values, y_pred, labels=classes))


import pymongo
import json
import jieba
import jieba.analyse

from bosonnlp import BosonNLP
from bloom_filter import BloomFilter
from pyquery import PyQuery as pq
from selenium import webdriver
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.common.exceptions import TimeoutException
from selenium.webdriver.common.by import By

'''
WeiboSpider.py:
                爬取微博健康相关的数据并保存在mongodb。用于DataAnalyzer来分析数据
'''

# 配置布隆过滤器，用于网址链接去重
filter = BloomFilter(filename='weibo_bloom', max_elements=100000, error_rate=0.001)

# 启动浏览器chrome模拟访问weibo.com, 设置selenium参数
browser = webdriver.Chrome()
wait = WebDriverWait(browser, 10)

# 连接MongoDB, 配置MongoDB参数
MONGO_TBL = 'healthy_info'
client = pymongo.MongoClient('localhost')
db = client['weibo']

# BosonNLP
nlp = BosonNLP('vtWvmE_M.33585.9k7qpYIWc6Aq')

# JiebaNLP设置参数
TOP_K = 8             # 抽取关键词的词数

# 要爬取的xhr对象页数，即要爬取多少页微博内容（首页+xhr页数)
XHR_PAGES = 30

class WeiboSpider:
    '''
    微博爬虫
    爬取过程分为两步：
    1. 爬取微博第一页内容，即分析html内容并提取正文内容，对应函数是__parse_url
    2. 爬取微博后面的异步内容，即分析xhr内容并提取正文内容，对应函数是__parse_xhr
    '''

    def __init__(self):
        self.run()

    def __parse_url(self, url):
        '''
        解析url内容，（html格式)并提取出里面正文内容并保存入数据库
        :param url: html对象的url
        :return: None
        '''
        try:
            browser.get(url)
        except TimeoutException:
            return None
        wait.until(EC.presence_of_all_elements_located((By.CSS_SELECTOR, '.WB_text.W_f14')))
        html = browser.page_source
        page = pq(html)
        items = page('.WB_text.W_f14').items()
        for item in items:
            content = item.text().replace('\n', '').replace('#', '').replace('【', '').replace('】', '')
            content = nlp.summary('', content)
            is_success = self.__save_to_mongo(content)
            if (is_success is True):
                filter.add(url)

    def __parse_xhr(self, url):
        '''
        解析xhr内容，（json格式)并提取出里面正文内容并保存入数据库
        :param url: xhr对象的url
        :return: None
        '''
        try:
            browser.get(url)
        except TimeoutException:
            return None
        # 等待标签 <pre>
        wait.until(EC.presence_of_all_elements_located((By.CSS_SELECTOR, 'pre')))
        # 分析网页内容
        html = browser.page_source
        doc = pq(html)
        content = doc.text()
        # json.load(content)将content从字符创变为字典
        # 接下来用get('data')将字典中的key为'data'的内容提取出来并保存在data中
        # data中保存的是字符串，保存的xhr的json内容，用于ajax加载增量的数据
        data = json.loads(content).get('data')
        # 将data用pyquery解析
        data_doc = pq(data)
        # 提取出微博该url的正文内容
        items = data_doc('.WB_text.W_f14').items()
        # 转为字典并保存到mongo中，添加到布隆过滤器中，用于以后的去重
        for item in items:
            # 首先先替换掉\n # 【 】
            content2 = item.text().replace('\n', '').replace('#', '').replace('【', '').replace('】', '')
            content3 = nlp.summary('', content2)  # 提取摘要
            # 如果是字母或数字则pass
            if (content3[0] >= '0' and content3[0] <= '9') or (content3[0] >= 'a' and content3[0] <= 'z'):
                continue
            is_success = self.__save_to_mongo(content3)
            if (is_success is True):
                filter.add(url)

    def __get_keywords(self, content):
        '''
        分析content并提取关键词存储在keywords并返回
        :param content: 文本内容
        :return: 一个存储关键词的列表 keywords_ls
        '''
        keywords_ls = jieba.analyse.extract_tags(content, topK=TOP_K, withWeight=False)    # 抽取关键词
        return keywords_ls

    def __save_to_mongo(self, content):
        '''
        将result结果存储到mongodb中
        :param result: 要存储入mongodb的数据
        :return: None
        '''
        if content is None:
            return False
        keywords = self.__get_keywords(content)
        result = {
            'content': content,             # 文章精简内容
            'keywords': keywords            # 关键词序列
        }
        try:
            if db[MONGO_TBL].insert_one(result):
                print("save to db success", result)
                return True
        except Exception:
            return False

    def run(self):
        '''
        爬虫开始运行
        :return: None
        '''
        # urls: 要爬取网址首页内容
        urls = [
                'https://weibo.com/shengmingshibao',            # 生命时报
                'https://weibo.com/dxydoctor',                  # 丁香医生
                'https://weibo.com/sinahealth',                 # 新浪健康
                'https://www.weibo.com/jiankangzhongguo',       # 健康中国
                'https://www.weibo.com/jiankangshibaoblog',     # 健康时报
                'https://weibo.com/cnhealthcare',               # 健康界
                'https://weibo.com/u/6387189723',               # 中国健康科普主场
                'https://weibo.com/u/6559157240',               # 健康类主流媒体
                'https://weibo.com/xiehehospital',              # 北京协和医院
                'https://weibo.com/wyjiankang',                 # 网易健康

        ]
        for url in urls:
            if url not in filter:
                self.__parse_url(url)

        ''' 要爬取网址xhr对象们 '''
        # xhr对象链接的公共部分
        part1 = 'https://weibo.com/p/aj/v6/mblog/mbloglist?ajwvr=6&domain=100206&is_hot=1&pagebar='
        # xhr对象链接不相同的部分
        xhr_part2 = [
            # 生命时报
            '&pl_name=Pl_Official_MyProfileFeed__28&id=1002061774057271&script_uri=/shengmingshibao&feed_type=0&page=1&pre_page=1&domain_op=100206&__rnd=1552821214799',
            # 丁香医生
            '&pl_name=Pl_Official_MyProfileFeed__25&id=1006062212518065&script_uri=/dxydoctor&feed_type=0&page=1&pre_page=1&domain_op=100606&__rnd=1552821290923',
            # 新浪健康
            '&pl_name=Pl_Official_MyProfileFeed__28&id=1002061268642527&script_uri=/sinahealth&feed_type=0&page=1&pre_page=1&domain_op=100206&__rnd=1552821037728',
            # 健康中国
            '&pl_name=Pl_Official_MyProfileFeed__25&id=1001062834480301&script_uri=/jiankangzhongguo&feed_type=0&page=1&pre_page=1&domain_op=100106&__rnd=1553266384513',
            # 健康时报
            '&pl_name=Pl_Official_MyProfileFeed__28&id=1002061411163204&script_uri=/jiankangshibaoblog&feed_type=0&page=1&pre_page=1&domain_op=100206&__rnd=1553266519100',
            # 健康界
            '&pl_name=Pl_Official_MyProfileFeed__25&id=1006062651153623&script_uri=/cnhealthcare&feed_type=0&page=1&pre_page=1&domain_op=100606&__rnd=1553269434318',
            # 中国健康科普主场
            '&pl_name=Pl_Official_MyProfileFeed__25&id=1006066387189723&script_uri=/u/6387189723&feed_type=0&page=1&pre_page=1&domain_op=100606&__rnd=1553269648205',
            # 健康类主流媒体
            '&pl_name=Pl_Official_MyProfileFeed__28&id=1002066559157240&script_uri=/u/6559157240&feed_type=0&page=1&pre_page=1&domain_op=100206&__rnd=1553269753856',
            # 北京协和医院
            '&pl_name=Pl_Official_MyProfileFeed__25&id=1001061654801402&script_uri=/xiehehospital&feed_type=0&page=1&pre_page=1&domain_op=100106&__rnd=1553270327353',
            # 网易健康
            'https://weibo.com/p/aj/v6/mblog/mbloglist?ajwvr=6&domain=100206&is_hot=1&pagebar=0&pl_name=Pl_Official_MyProfileFeed__28&id=1002065341241739&script_uri=/wyjiankang&feed_type=0&page=1&pre_page=1&domain_op=100206&__rnd=1553270598357',
        ]

        xhr_part2_len = len(xhr_part2)
        for i in range(0, XHR_PAGES):
            for j in range(0, xhr_part2_len):
                url = part1+str(i)+xhr_part2[j]
                if url not in filter:
                    self.__parse_xhr(url)

if __name__ == '__main__':
    weibo = WeiboSpider()


import discord
from discord.ext import commands
from bs4 import BeautifulSoup
from requests.utils import quote
import aiohttp

class CardSkillMixedException(Exception):
        def __init__(self, message):
            super().__init__(message)

class HTMLParseException(Exception):
        def __init__(self, message):
            super().__init__(message)

class DuelLinksLinkNotFoundException(Exception):
        def __init__(self, message):
            super().__init__(message)

class DuelLinksCog:

    def __init__(self, bot):
        self.bot = bot

    @commands.command(pass_context=True, aliases=["s"])
    async def skill(self, ctx, *skill_names):
        """Show a Yu-Gi-Oh! Duel Links Skill.
        Surround your card name(s) with apostrophes (') for exact matches.
        Use a comma-separated list for multiple skills."""

        joined_skill_names = ' '.join(skill_names)
        if not any([letter.isalpha() for letter in joined_skill_names]):
            await self.bot.send_cmd_help(ctx)
            return

        skill_list = joined_skill_names.split(",")
        for skill_name in skill_list:
            if not skill_name.isspace():
                try:
                    skill_link = await self._get_duel_links_link(skill_name.strip())
                except DuelLinksLinkNotFoundException as e:
                    await self.bot.say(e)
                    return
                try:
                    skill_details = await self._get_skill_details(skill_link)
                except CardSkillMixedException as e:
                    await self.bot.say(e)
                    return
                except HTMLParseException as e:
                    await self.bot.say("%s\n%s" % (e, card_link))
                    return
                skill_title = "**`%s`**" % skill_details["Name"]
                skill_description = "```\n%s```" % skill_details["Description"]
                await self.bot.say("%s\n%s\n%s" % (skill_title, skill_description, skill_link))

    @commands.command(pass_context=True, aliases=["c"])
    async def card(self, ctx, *card_names):
        """Show a Yu-Gi-Oh! card.
        Surround your card name(s) with apostrophes (') for exact matches.
        Use a comma-separated list for multiple cards."""

        joined_card_names = ' '.join(card_names)
        if not any([letter.isalpha() for letter in joined_card_names]):
            await self.bot.send_cmd_help(ctx)
            return

        card_list = joined_card_names.split(",")
        for card_name in card_list:
            if not card_name.isspace():
                try:
                    card_link = await self._get_duel_links_link(card_name.strip())
                except DuelLinksLinkNotFoundException as e:
                    await self.bot.say(e)
                    return
                try:
                    card_details = await self._get_card_details(card_link)
                except CardSkillMixedException as e:
                    await self.bot.say(e)
                    return
                except HTMLParseException as e:
                    await self.bot.say("%s\n%s" % (e, card_link))
                    return
                bot_message = []

                title_message = "**`%s" % card_details["Name"]
                if "Level" in card_details:
                    title_message += " %s" % card_details["Level"]
                if "Rarity" in card_details:
                    title_message += " %s" % card_details["Rarity"]
                title_message += "`**"
                bot_message.append(title_message)

                if "Attribute" in card_details and "Monster Type" in card_details:
                    bot_message.append("**`Attribute: `**`%s` **`Type: `**`%s`" % (card_details["Attribute"], card_details["Monster Type"]))

                bot_message.append("```\n%s```" % card_details["Description"])

                if "ATK" in card_details and "DEF" in card_details:
                    bot_message.append("**`ATK: `**`%s` **`DEF: `**`%s`" % (card_details["ATK"], card_details["DEF"]))

                bot_message.append(card_link)

                await self.bot.say("\n".join(bot_message))

    async def _get_duel_links_link(self, query):
        query_is_quoted = query.startswith("'") and query.endswith("'")
        google_search = "http://www.google.com/search?q=" + quote("duel links %s" % query)
        browser_headers = {
            "User-Agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/47.0.2526.106 Safari/537.36"
        }
        async with aiohttp.get(google_search, headers=browser_headers) as response:
            google_search_soup = BeautifulSoup(await response.text(), "html.parser")

        h3_text = list(map(lambda h3: h3.text, google_search_soup.find_all("h3")))
        cite_text = list(map(lambda cite: cite.text, google_search_soup.find_all("cite")))
        search_results = zip(h3_text, cite_text)
        for search_results in search_results:
            if len(search_results) is not 2:
                continue
            if not search_results[1].startswith("duellinks.gamea.co"):
                continue
            if query_is_quoted:
                search_result_card_name = search_results[0].split("|")[0].strip()
                if query.strip("'").lower() == search_result_card_name.lower():
                    return "http://%s" % search_results[1]
                continue
            return "http://%s" % search_results[1]
        raise DuelLinksLinkNotFoundException("Could not find the Duel Links link.")

    async def _get_card_details(self, card_link):
        async with aiohttp.get(card_link) as response:
            duel_links_soup = BeautifulSoup(await response.text(), "html.parser")

        card_details = {}
        card_details["Name"] = duel_links_soup.find("title").text.split("|")[0].strip()
        card_details["Description"] = duel_links_soup.find("p").text

        if "(skill)" in card_details["Name"]:
            raise CardSkillMixedException("%s is a skill - not a card." % card_details["Name"][:-8])

        # get the rarity
        rarity_levels = ["[N]", "[R]", "[SR]", "[UR]"]
        found_rarity = False
        article_parts = duel_links_soup.findAll("div", {"class": "article-part"})
        for i in range(0, len(article_parts)):
            # need to search case-insensitive for the how to get/rarity section
            rarity_location_title = article_parts[i].text.lower()
            if "rarity" in rarity_location_title or "how to get" in rarity_location_title:
                rarity_location = article_parts[i + 1]
                for rarity_table_td in rarity_location.find_all("td"):
                    rarity_table_cell_text = rarity_table_td.get_text()
                    for rarity_level in rarity_levels:
                        # search everything inside the <td> via get_text()
                        if rarity_level in rarity_table_cell_text:
                            card_details["Rarity"] = rarity_level
                            found_rarity = True
                            break
            if found_rarity:
                break

        # get other card details
        all_tr = duel_links_soup.find_all("tr")
        for tr in all_tr:
            all_td = tr.find_all("td")
            if not all_td:
                continue
            th = tr.find_all("th")
            key_td = th[0] if th else all_td[0]
            value_td = all_td[-1]
            if key_td.text in ["Rating", "Pack"]:
                return card_details
            elif key_td.text in ["Passcode"]:
                continue
            card_details[key_td.text] = value_td.text
        raise HTMLParseException("The HTML retrieved from the Duel Links site was fucked.")

    async def _get_skill_details(self, skill_link):
        async with aiohttp.get(skill_link) as response:
            duel_links_soup = BeautifulSoup(await response.text(), "html.parser")
        skill_details = {}
        skill_details["Name"] = duel_links_soup.find("title").text.split("|")[0].strip()

        if "(skill)" not in skill_details["Name"]:
            raise CardSkillMixedException("%s is a card - not a skill." % skill_details["Name"])

        article_parts = duel_links_soup.findAll("div", {"class": "article-part"})
        for i in range(0, len(article_parts)):
            if article_parts[i].text.strip() == "Description":
                skill_details["Description"] = article_parts[i + 1].text.strip()
                break
        return skill_details

def setup(bot):
    bot.add_cog(DuelLinksCog(bot))


# -*- coding: utf-8 -*-

from flask.ext.rq import job
from tweepy import OAuthHandler, API
import bitlyapi
import urllib2
import json
import re
from hasjob import app

@job('hasjob')
def tweet(title, url, location=None, parsed_location=None):
    auth = OAuthHandler(app.config['TWITTER_CONSUMER_KEY'], app.config['TWITTER_CONSUMER_SECRET'])
    auth.set_access_token(app.config['TWITTER_ACCESS_KEY'], app.config['TWITTER_ACCESS_SECRET'])
    api = API(auth)
    urllength = 23  # Current Twitter standard for HTTPS (as of Oct 2014)
    maxlength = 140 - urllength - 1  # == 116
    locationtag = u''
    if parsed_location:
        locationtags = []
        for token in parsed_location.get('tokens', []):
            if 'geoname' in token and 'token' in token:
                locname = token['token'].strip()
                if locname:
                    locationtags.append(u'#' + locname.title().replace(u' ', ''))
        locationtag = u' '.join(locationtags)
        if locationtag:
            maxlength -= len(locationtag) + 1
    if not locationtag and location:
        # Make a hashtag from the first word in the location. This catches
        # locations like 'Anywhere' which have no geonameid but are still valid
        locationtag = u'#' + re.split('\W+', location)[0]
        maxlength -= len(locationtag) + 1

    if len(title) > maxlength:
        text = title[:maxlength - 1] + u'…'
    else:
        text = title[:maxlength]
    text = text + ' ' + url  # Don't shorten URLs, now that there's t.co
    if locationtag:
        text = text + ' ' + locationtag
    api.update_status(text)

def shorten(url):
    if app.config['BITLY_KEY']:
        b = bitlyapi.BitLy(app.config['BITLY_USER'], app.config['BITLY_KEY'])
        res = b.shorten(longUrl=url)
        return res['url']
    else:
        req = urllib2.Request("https://www.googleapis.com/urlshortener/v1/url",
            headers={"Content-Type": "application/json"},
            data=json.dumps({'longUrl': url}))
        request_result = urllib2.urlopen(req)
        result = request_result.read()
        result_json = json.loads(result)
        return result_json['id']


from wx import wx
import wx.grid as wxgrid
import sqlite3
import AddTableDialog
import EditTableProperties

class CreateTableTab(wx.Panel):
    def __init__(self, parent):
        wx.Panel.__init__(self, parent)

        self.toolbar_ADDTABLE = wx.NewId()
        self.toolbar_DELTABLE = wx.NewId()
        self.toolbar_ADDROW = wx.NewId()
        self.toolbar_DELROW = wx.NewId()
        self.toolbar_ADDCOL = wx.NewId()
        self.toolbar_DELCOL = wx.NewId()
        self.toolbar_UPDATETABLE = wx.NewId()
        self.toolbar_BOLDSEL = wx.NewId()
        self.toolbar_SLANTSEL = wx.NewId()
        self.toolbar_UNDERSEL = wx.NewId()
        self.toolbar_EDITPROP = wx.NewId()
        self.SetBackgroundColour(wx.SystemSettings.GetColour(wx.SYS_COLOUR_3DFACE))
        self.gridPanel = wx.Panel(self)
        self.tableListBox = wx.ListBox(self, -1, size=wx.Size(100, -1), style=wx.BORDER_SUNKEN | wx.LB_SINGLE | wx.LB_NEEDED_SB)
        self.sizer = wx.BoxSizer(wx.VERTICAL)
        self.gridPanel.SetSizer(self.sizer)
        self.tableToolbar = self.MakeTableToolbar()
        self.mainBox = wx.BoxSizer(wx.VERTICAL)
        self.botBox = wx.BoxSizer(wx.HORIZONTAL)
        self.topBox = wx.BoxSizer(wx.HORIZONTAL)
        self.topBox.Add(self.tableToolbar, 1, wx.ALL | wx.ALIGN_LEFT | wx.EXPAND)
        self.botBox.Add(self.tableListBox, 0, wx.ALL | wx.ALIGN_LEFT | wx.EXPAND)
        self.botBox.Add(self.gridPanel, 1, wx.EXPAND)
        self.mainBox.Add(self.topBox, 0, wx.ALL | wx.ALIGN_LEFT | wx.EXPAND)
        self.mainBox.Add(self.botBox, 1, wx.GROW | wx.EXPAND)
        self.SetSizer(self.mainBox)
        self.tableList = []

        wx.EVT_LISTBOX(self, self.tableListBox.GetId(), self.OnSelectedTableItem)

    def MakeTableToolbar(self):
        tb = wx.ToolBar(self, -1, style=wx.TB_FLAT | wx.NO_BORDER)
        tb.AddTool(self.toolbar_ADDTABLE, wx.Bitmap("images/list-add.png", wx.BITMAP_TYPE_PNG), shortHelpString="Add Table")
        tb.AddTool(self.toolbar_DELTABLE, wx.Bitmap("images/list-remove.png", wx.BITMAP_TYPE_PNG), shortHelpString="Remove Selected Table")
        tb.AddSeparator()
        tb.AddTool(self.toolbar_ADDROW, wx.Bitmap("images/row-add.png", wx.BITMAP_TYPE_PNG), shortHelpString="Insert Row")
        tb.AddTool(self.toolbar_DELROW, wx.Bitmap("images/row-del.png", wx.BITMAP_TYPE_PNG), shortHelpString="Delete Row")
        tb.AddTool(self.toolbar_ADDCOL, wx.Bitmap("images/col-add.png", wx.BITMAP_TYPE_PNG), shortHelpString="Insert Column")
        tb.AddTool(self.toolbar_DELCOL, wx.Bitmap("images/col-del.png", wx.BITMAP_TYPE_PNG), shortHelpString="Delete Column")
        tb.AddSeparator()
        tb.AddTool(self.toolbar_BOLDSEL, wx.Bitmap("images/format-text-bold.png", wx.BITMAP_TYPE_PNG), isToggle = True, shortHelpString="Bold")
        tb.AddTool(self.toolbar_SLANTSEL, wx.Bitmap("images/format-text-italic.png", wx.BITMAP_TYPE_PNG), isToggle = True, shortHelpString="Italic")
        tb.AddTool(self.toolbar_UNDERSEL, wx.Bitmap("images/format-text-underline.png", wx.BITMAP_TYPE_PNG), isToggle = True, shortHelpString="Underline")
        tb.AddSeparator()
        tb.AddTool(self.toolbar_EDITPROP, wx.Bitmap("images/document-properties.png", wx.BITMAP_TYPE_PNG), shortHelpString="Table Properties")
        tb.AddTool(self.toolbar_UPDATETABLE, wx.Bitmap("images/document-save.png", wx.BITMAP_TYPE_PNG), shortHelpString="Update Table Content")
        tb.EnableTool(self.toolbar_ADDTABLE, True)
        tb.EnableTool(self.toolbar_DELTABLE, False)
        tb.EnableTool(self.toolbar_ADDCOL, False)
        tb.EnableTool(self.toolbar_ADDROW, False)
        tb.EnableTool(self.toolbar_DELCOL, False)
        tb.EnableTool(self.toolbar_DELROW, False)
        tb.EnableTool(self.toolbar_BOLDSEL, False)
        tb.EnableTool(self.toolbar_SLANTSEL, False)
        tb.EnableTool(self.toolbar_UPDATETABLE, False)
        tb.EnableTool(self.toolbar_UNDERSEL, False)
        tb.EnableTool(self.toolbar_EDITPROP, False)

        wx.EVT_TOOL(self, self.toolbar_ADDTABLE, self.AddTable)
        wx.EVT_TOOL(self, self.toolbar_DELTABLE, self.DelTable)
        wx.EVT_TOOL(self, self.toolbar_ADDROW, self.AddRow)
        wx.EVT_TOOL(self, self.toolbar_DELROW, self.DelRow)
        wx.EVT_TOOL(self, self.toolbar_ADDCOL, self.AddCol)
        wx.EVT_TOOL(self, self.toolbar_DELCOL, self.DelCol)
        wx.EVT_TOOL(self, self.toolbar_BOLDSEL, self.BoldSelection)
        wx.EVT_TOOL(self, self.toolbar_SLANTSEL, self.SlantSelection)
        wx.EVT_TOOL(self, self.toolbar_UPDATETABLE, self.UpdateTable)
        wx.EVT_TOOL(self, self.toolbar_UNDERSEL, self.UnderSelection)
        wx.EVT_TOOL(self, self.toolbar_EDITPROP, self.EditProperties)

        tb.Realize()

        return tb

    def OnSelectedTableItem(self, event):
        if(self.tableListBox.GetSelection() != -1):
            self.tableToolbar.EnableTool(self.toolbar_DELTABLE, True)
            self.tableToolbar.EnableTool(self.toolbar_ADDCOL, True)
            self.tableToolbar.EnableTool(self.toolbar_ADDROW, True)
            self.tableToolbar.EnableTool(self.toolbar_DELCOL, True)
            self.tableToolbar.EnableTool(self.toolbar_DELROW, True)
            self.tableToolbar.EnableTool(self.toolbar_BOLDSEL, True)
            self.tableToolbar.EnableTool(self.toolbar_SLANTSEL, True)
            self.tableToolbar.EnableTool(self.toolbar_UNDERSEL, True)
            self.tableToolbar.EnableTool(self.toolbar_UPDATETABLE, True)
            self.tableToolbar.EnableTool(self.toolbar_EDITPROP, True)

            # start the function to load values
            tmpString = self.tableListBox.GetStringSelection()
            tmpString2 = tmpString.split(".", 1)
            self.TopLevelParent.DisplayDebug(self.TopLevelParent.currentTableDict[int(tmpString2[0])][0])
            self.rawtabledata = self.TopLevelParent.currentTableDict[int(tmpString2[0])][0]
            self.tableRows = self.rawtabledata.split("||")
            self.tableList[:] = []
            for row in self.tableRows:
                self.tableList.append(row.split("::"))
            if(self.gridPanel.GetChildren() != []):
                self.gridPanel.DestroyChildren()
            self.tableGrid = wxgrid.Grid(self.gridPanel)
            wxgrid.EVT_GRID_SELECT_CELL(self, self.OnSelectCell)
            self.tableGrid.CreateGrid(len(self.tableList), len(self.tableList[0]))
            self.tableGrid.SetRowLabelSize(0)
            self.tableGrid.SetColLabelSize(0)
            self.sizer.Add(self.tableGrid, 1, wx.GROW | wx.EXPAND)
            self.gridPanel.Layout()
            # need to loop over tablelist to populate grid cells.
            for i in range(0, len(self.tableList)):
                for j in range(0, len(self.tableList[i])):
                    self.tableGrid.SetCellValue(i, j, self.tableList[i][j])

    def AddTable(self, event):
        self.addTableDialog = AddTableDialog.AddTableDialog(self, -1, "Add Table to Course", size=(550, 200), style=wx.DEFAULT_DIALOG_STYLE)
        self.addTableDialog.CenterOnParent()

        if(self.addTableDialog.ShowModal() == wx.ID_OK):
            self.TopLevelParent.DisplayDebug("you pressed OK")
            self.layoutValue = self.addTableDialog.tableAltText.GetValue() + "::" + self.addTableDialog.tableCaptionText.GetValue()
            con = sqlite3.connect(self.TopLevelParent.database)
            cur = con.cursor()
            cur.execute("INSERT INTO ResourceTable(resourceTypeID, resourceData, layout, courseID) VALUES(3, '::::::::::||::::::::::||::::::::::||::::::::::||::::::::::', ?, ?)", (self.layoutValue, self.TopLevelParent.currentCourseID))
            lastID = cur.lastrowid
            con.commit()
            con.close()
            self.TopLevelParent.currentTableDict = self.TopLevelParent.RefreshListBox(self.tableListBox, self.TopLevelParent.currentCourseID, 3, None)
            if(self.gridPanel.GetChildren() != []):
                self.gridPanel.DestroyChildren()
            self.tableGrid = wxgrid.Grid(self.gridPanel)
            wxgrid.EVT_GRID_SELECT_CELL(self, self.OnSelectCell)
            self.tableGrid.CreateGrid(5, 5)
            self.tableGrid.SetRowLabelSize(0)
            self.tableGrid.SetColLabelSize(0)
            self.sizer.Add(self.tableGrid, 1, wx.GROW | wx.EXPAND)
            self.gridPanel.Layout()
            self.tableListBox.SetSelection(self.tableListBox.FindString(str(lastID) + ". " + self.addTableDialog.tableCaptionText.GetValue()))
        else:
            self.TopLevelParent.DisplayDebug("you pressed cancel")

        self.addTableDialog.Destroy()

    def EditProperties(self, event):
        self.TopLevelParent.DisplayDebug("edit here")
        self.editTableProperties = EditTableProperties.EditTableProperties(self, -1, "Edit Table Properties", size=(550, 200), style=wx.DEFAULT_DIALOG_STYLE)
        self.editTableProperties.CenterOnParent()
        tmpString = self.tableListBox.GetStringSelection() # get selected item
        tmpId = int(tmpString.split(".", 1)[0]) # extract resourceID
        con = sqlite3.connect(self.TopLevelParent.database)
        cur = con.cursor()
        cur.execute("SELECT layout FROM ResourceTable WHERE resourceID = (?)", (tmpId,))
        for row in cur:
            selectedLayout = row[0].split("::")
            self.editTableProperties.tableAltText.SetValue(selectedLayout[0])
            self.editTableProperties.tableCaptionText.SetValue(selectedLayout[1])
        con.close()

        if(self.editTableProperties.ShowModal() == wx.ID_OK):
            self.layoutValue = self.editTableProperties.tableAltText.GetValue() + "::" + self.editTableProperties.tableCaptionText.GetValue()
            con = sqlite3.connect(self.TopLevelParent.database)
            cur = con.cursor()
            cur.execute("UPDATE ResourceTable SET layout = (?) WHERE resourceID = (?)", (self.layoutValue, tmpId))
            con.commit()
            con.close()
            self.TopLevelParent.currentTableDict = self.TopLevelParent.RefreshListBox(self.tableListBox, self.TopLevelParent.currentCourseID, 3, None)
            self.TopLevelParent.DisplayDebug("ok")
        else:
            self.TopLevelParent.DisplayDebug("cancel")

        self.editTableProperties.Destroy()

    def DelTable(self, event):
        tmpString = self.tableListBox.GetStringSelection()
        tmpString2 = tmpString.split(".", 1)
        con = sqlite3.connect(self.TopLevelParent.database)
        cur = con.cursor()
        cur.execute("DELETE FROM ResourceTable WHERE resourceID = (?)", (int(tmpString2[0]),))
        con.commit()
        con.close()
        self.TopLevelParent.currentTableDict = self.TopLevelParent.RefreshListBox(self.tableListBox, self.TopLevelParent.currentCourseID, 3, None)
        self.TopLevelParent.DisplayDebug("table ID " + tmpString2[0] + " deleted")

    def AddRow(self, event):
        if(self.tableGrid.GetSelectionBlockTopLeft() != []):
            self.tableGrid.InsertRows(self.tableGrid.GetSelectionBlockTopLeft()[0][0], (self.tableGrid.GetSelectionBlockBottomRight()[0][0] - self.tableGrid.GetSelectionBlockTopLeft()[0][0] + 1))
        else:
            self.tableGrid.InsertRows(self.tableGrid.GetGridCursorRow(), 1)

    def DelRow(self, event):
        if(self.tableGrid.GetSelectionBlockTopLeft() != []):
            self.tableGrid.DeleteRows(self.tableGrid.GetSelectionBlockTopLeft()[0][0], (self.tableGrid.GetSelectionBlockBottomRight()[0][0] - self.tableGrid.GetSelectionBlockTopLeft()[0][0] + 1))
        else:
            self.tableGrid.DeleteRows(self.tableGrid.GetGridCursorRow(), 1)

    def AddCol(self, event):
        if(self.tableGrid.GetSelectionBlockTopLeft() != []):
            self.tableGrid.InsertCols(self.tableGrid.GetSelectionBlockTopLeft()[0][1], (self.tableGrid.GetSelectionBlockBottomRight()[0][1]) - self.tableGrid.GetSelectionBlockTopLeft()[0][1] + 1)
        else:
            self.tableGrid.InsertCols(self.tableGrid.GetGridCursorCol(), 1)

    def DelCol(self, event):
        if(self.tableGrid.GetSelectionBlockTopLeft() != []):
            self.tableGrid.DeleteCols(self.tableGrid.GetSelectionBlockTopLeft()[0][1], (self.tableGrid.GetSelectionBlockBottomRight()[0][1] - self.tableGrid.GetSelectionBlockTopLeft()[0][1] + 1))
        else:
            self.tableGrid.DeleteCols(self.tableGrid.GetGridCursorCol(), 1)

    def BoldSelection(self, event):
        if(self.tableGrid.GetSelectionBlockTopLeft() != []):
            # loop to apply bold to all selected cells
            for i in range(self.tableGrid.GetSelectionBlockTopLeft()[0][0], self.tableGrid.GetSelectionBlockBottomRight()[0][0]+1):
                for j in range(self.tableGrid.GetSelectionBlockTopLeft()[0][1], self.tableGrid.GetSelectionBlockBottomRight()[0][1]+1):
                    tmpFont = self.tableGrid.GetCellFont(i, j)
                    if (self.tableGrid.GetCellFont(i, j).GetWeight() == wx.NORMAL):
                        tmpFont.SetWeight(wx.FONTWEIGHT_BOLD)
                    else:
                        tmpFont.SetWeight(wx.FONTWEIGHT_NORMAL)
                    self.tableGrid.SetCellFont(i, j, tmpFont)
        else: #apply bold to currently selected cell
            tmpFont = self.tableGrid.GetCellFont(self.tableGrid.GetGridCursorRow(), self.tableGrid.GetGridCursorCol())
            if(self.tableGrid.GetCellFont(self.tableGrid.GetGridCursorRow(), self.tableGrid.GetGridCursorCol()).GetWeight() == wx.NORMAL):
                tmpFont.SetWeight(wx.FONTWEIGHT_BOLD)
            else:
                tmpFont.SetWeight(wx.FONTWEIGHT_NORMAL)
            self.tableGrid.SetCellFont(self.tableGrid.GetGridCursorRow(), self.tableGrid.GetGridCursorCol(), tmpFont)
        self.tableGrid.Refresh() # update style immediately

    def SlantSelection(self, event):
        if(self.tableGrid.GetSelectionBlockTopLeft() != []):
            # "loop to apply bold to all selected cells"
            for i in range(self.tableGrid.GetSelectionBlockTopLeft()[0][0], self.tableGrid.GetSelectionBlockBottomRight()[0][0]+1):
                for j in range(self.tableGrid.GetSelectionBlockTopLeft()[0][1], self.tableGrid.GetSelectionBlockBottomRight()[0][1]+1):
                    tmpFont = self.tableGrid.GetCellFont(i, j)
                    if(self.tableGrid.GetCellFont(i, j).GetStyle() == wx.NORMAL):
                        tmpFont.SetStyle(wx.FONTSTYLE_ITALIC)
                    else:
                        tmpFont.SetStyle(wx.FONTSTYLE_NORMAL)
                    self.tableGrid.SetCellFont(i, j, tmpFont)
        else:
            tmpFont = self.tableGrid.GetCellFont(self.tableGrid.GetGridCursorRow(), self.tableGrid.GetGridCursorCol())
            if(self.tableGrid.GetCellFont(self.tableGrid.GetGridCursorRow(), self.tableGrid.GetGridCursorCol()).GetStyle() == wx.NORMAL):
                tmpFont.SetStyle(wx.FONTSTYLE_ITALIC)
            else:
                tmpFont.SetStyle(wx.FONTSTYLE_NORMAL)
            self.tableGrid.SetCellFont(self.tableGrid.GetGridCursorRow(), self.tableGrid.GetGridCursorCol(), tmpFont)
        self.tableGrid.Refresh() # update style immediately

    def UnderSelection(self, event):
        if(self.tableGrid.GetSelectionBlockTopLeft() != []):
            # loop to apply underline to all selected cells
            for i in range(self.tableGrid.GetSelectionBlockTopLeft()[0][0], self.tableGrid.GetSelectionBlockBottomRight()[0][0]+1):
                for j in range(self.tableGrid.GetSelectionBlockTopLeft()[0][1], self.tableGrid.GetSelectionBlockBottomRight()[0][1]+1):
                    tmpFont = self.tableGrid.GetCellFont(i, j)
                    if(self.tableGrid.GetCellFont(i, j).GetUnderlined() == False):
                        tmpFont.SetUnderlined(True)
                    else:
                        tmpFont.SetUnderlined(False)
                    self.tableGrid.SetCellFont(i, j, tmpFont)
        else:
            tmpFont = self.tableGrid.GetCellFont(self.tableGrid.GetGridCursorRow(), self.tableGrid.GetGridCursorCol())
            if(self.tableGrid.GetCellFont(self.tableGrid.GetGridCursorRow(), self.tableGrid.GetGridCursorCol()).GetUnderlined() == False):
                tmpFont.SetUnderlined(True)
            else:
                tmpFont.SetUnderlined(False)
            self.tableGrid.SetCellFont(self.tableGrid.GetGridCursorRow(), self.tableGrid.GetGridCursorCol(), tmpFont)
        self.tableGrid.Refresh() # update style immediately

    def OnSelectCell(self, event): # check cell font status.
        tmpFont= self.tableGrid.GetCellFont(event.GetRow(), event.GetCol())
        if(tmpFont.GetStyle() == wx.ITALIC):
            self.tableToolbar.ToggleTool(self.toolbar_SLANTSEL, True)
        else:
            self.tableToolbar.ToggleTool(self.toolbar_SLANTSEL, False)
        if(tmpFont.GetWeight() == wx.BOLD):
            self.tableToolbar.ToggleTool(self.toolbar_BOLDSEL, True)
        else:
            self.tableToolbar.ToggleTool(self.toolbar_BOLDSEL, False)
        if(tmpFont.GetUnderlined() == True):
            self.tableToolbar.ToggleTool(self.toolbar_UNDERSEL, True)
        else:
            self.tableToolbar.ToggleTool(self.toolbar_UNDERSEL, False)

        event.Skip() # allows other events to continue processing

    def UpdateTable(self, event):
        self.tableGrid.HideCellEditControl()
        tmpData = ""
        self.TopLevelParent.DisplayDebug("save cell values to db")
        for i in range(0, self.tableGrid.GetNumberRows()):
            if(i > 0): tmpData += "||"
            for j in range(0, self.tableGrid.GetNumberCols()):
                if(j > 0 and j < self.tableGrid.GetNumberCols()): tmpData += "::"
                tmpData += self.tableGrid.GetCellValue(i, j)

        self.TopLevelParent.DisplayDebug(tmpData)

        curItem = self.tableListBox.GetSelection()
        tmpString = self.tableListBox.GetStringSelection()
        tmpString2 = tmpString.split(".", 1)
        con = sqlite3.connect(self.TopLevelParent.database)
        cur = con.cursor()
        cur.execute("UPDATE ResourceTable SET resourceData = (?) where resourceID = (?)", (tmpData, int(tmpString2[0]),))
        con.commit()
        con.close()
        self.TopLevelParent.currentTableDict = self.TopLevelParent.RefreshListBox(self.tableListBox, self.TopLevelParent.currentCourseID, 3, None)
        self.tableListBox.SetSelection(curItem)


# encoding=utf-8
# Author: Yu-Lun Chiang
# Description: Get news

import logging

from bs4 import BeautifulSoup

from .base import BaseMediaNewsCrawler
from ..struct import NewsStruct

logger = logging.getLogger(__name__)

class WantRich(BaseMediaNewsCrawler):
    """Web Crawler for WantRich News"""

    def getInfo(self, link: str) -> NewsStruct:
        return super().getInfo(link)

    @staticmethod
    def _get_content(
        soup: BeautifulSoup,
    ) -> str:

        content_list = soup.find(
            class_="article-body", itemprop="articleBody"
        ).find_all("p")
        content = "\n".join([c.text for c in content_list])
        logger.debug(f"CONTENT:\n {content}")
        return content


"""Functions that deal with orders logic"""
from datetime import datetime
import candy_delivery_app.database.db as db

def is_order_suitable(courier, order):
    """
    Check whether the order is suitable to the courier
    """

    max_weight = db.get_courier_type_weight(
        db.get_courier_type_id(courier['courier_type'])
    )

    if order['weight'] > max_weight:
        return False

    if order['region'] not in courier['regions']:
        return False

    current_year = int(datetime.now().year)
    current_month = int(datetime.now().month)
    current_day = int(datetime.now().day)

    for working_hour in map(lambda x: x.split('-'),
                            courier['working_hours']):

        work_start = datetime(year=current_year,
                              month=current_month,
                              day=current_day,
                              hour=int(working_hour[0][:2]),
                              minute=int(working_hour[0][3:])
                              )
        work_end = datetime(year=current_year,
                            month=current_month,
                            day=current_day,
                            hour=int(working_hour[1][:2]),
                            minute=int(working_hour[1][3:]),
                            )

        for delivery_hour in map(lambda x: x.split('-'),
                                 order['delivery_hours']):
            delivery_start = datetime(year=current_year,
                                      month=current_month,
                                      day=current_day,
                                      hour=int(delivery_hour[0][:2]),
                                      minute=int(delivery_hour[0][3:]),
                                      )
            delivery_end = datetime(year=current_year,
                                    month=current_month,
                                    day=current_day,
                                    hour=int(delivery_hour[1][:2]),
                                    minute=int(delivery_hour[1][3:]),
                                    )

            if work_start < delivery_end and work_end > delivery_start:
                return True
    return False