# -*- coding: utf-8 -*-
import os

from pymongo import MongoClient

class DBManager(object):
    db = None
    connection = None

    @classmethod
    def get_db(cls):
        if not cls.db:
            if not cls.connection:
                cls.connection = MongoClient(os.environ['CRAWLER_MONGO_URL'])

            if os.environ['PYTHON_ENV'] in ['development', 'staging', 'production']:
                cls.db = cls.connection.appbee
                print('Use appbee db for', os.environ['PYTHON_ENV'], 'environment')
            elif os.environ['PYTHON_ENV'] in ['test']:
                cls.db = cls.connection.test
                print('Use test db for', os.environ['PYTHON_ENV'], 'environment')
            else:
                print('Wrong environment:', os.environ['PYTHON_ENV'])

        return cls.db

    @classmethod
    def close(cls):
        if cls.connection:
            cls.connection.close()

    @classmethod
    def get_uncrawled_package_names_without_error_code(cls):
        db = cls.get_db()
        uncrawled_package_names = db['uncrawled-apps'].distinct('packageName', {'errCode': {'$exists': False}})
        other_apps_package_names = db['other-apps'].distinct('packageName')

        return list(filter(lambda package_name: package_name not in other_apps_package_names, uncrawled_package_names))

    @classmethod
    def update_error_code_of_uncrawled_app(cls, package_name, error_code):
        print('### Update errCode:' + str(error_code) + ' of ' + str(package_name) + 'in Uncrawled Apps ###')
        db = cls.get_db()

        db['uncrawled-apps'].update_one({'packageName': package_name}, {"$set": {'errCode': error_code}}, upsert=False)

    @classmethod
    def delete_uncrawled_app(cls, package_name):
        print('### Delete ' + str(package_name) + ' from Uncrawled Apps ###')
        db = cls.get_db()
        db['uncrawled-apps'].delete_one({'packageName': package_name})

    @classmethod
    def update_app_usages(cls, item):
        app_info = {
            'appName': item['appName'],
            'categoryId': item['categoryId1'],
            'categoryName': item['categoryName1'],
            'developer': item['developer'],
            'iconUrl': item['iconUrl'],
        }

        DBManager.get_db()['app-usages'].update_many({'packageName': item['packageName']}, {'$set': app_info})

    @classmethod
    def upsert_app(cls, item):
        print('### Upsert ' + str(item['packageName']) + ' to Apps ###')
        db = cls.get_db()
        db['apps'].update_one({'packageName': item['packageName']}, {'$set': dict(item)}, upsert=True)
        cls.update_app_usages(item)

    @classmethod
    def upsert_category(cls, item):
        print('### Upsert ' + str(item['id']) + ' to Categories ###')
        db = cls.get_db()
        db['categories'].update_one({'id': item['id']}, {'$set': dict(item)}, upsert=True)

    @classmethod
    def upsert_other_app(cls, item):
        print('### Upsert ' + str(item['packageName']) + ' to Other App ###')
        db = cls.get_db()
        db['other-apps'].update_one({'packageName': item['packageName']}, {'$set': dict(item)}, upsert=True)

    @classmethod
    def get_package_names_to_update_info(cls):
        db = cls.get_db()
        package_names = db['app-usages'].distinct('packageName')
        return package_names


import detectron2.utils.comm as comm
from detectron2.checkpoint import DetectionCheckpointer
from detectron2.config import get_cfg
from detectron2.engine import default_argument_parser, default_setup, launch

from adapteacher import add_ateacher_config
from adapteacher.engine.trainer import ATeacherTrainer

# hacky way to register
from adapteacher.modeling.meta_arch.rcnn import TwoStagePseudoLabGeneralizedRCNN, DAobjTwoStagePseudoLabGeneralizedRCNN
# from adapteacher.modeling.meta_arch.vgg import build_vgg_backbone  # noqa
from adapteacher.modeling.proposal_generator.rpn import PseudoLabRPN
from adapteacher.modeling.roi_heads.roi_heads import StandardROIHeadsPseudoLab
import adapteacher.data.datasets.builtin

from adapteacher.modeling.meta_arch.ts_ensemble import EnsembleTSModel
from detectron2.engine import DefaultPredictor
from detectron2.utils.visualizer import Visualizer
from detectron2.data import MetadataCatalog

import cv2
import torch
import numpy as np

from fvcore.transforms.transform import (
    NoOpTransform,
)
from detectron2.data.transforms.augmentation_impl import ResizeShortestEdge

def setup(args):
    """
    Create configs and perform basic setups.
    """
    cfg = get_cfg()
    add_ateacher_config(cfg)
    cfg.merge_from_file(args.config_file)
    cfg.merge_from_list(args.opts)
    cfg.MODEL.ROI_HEADS.SCORE_THRESH_TEST = 0.3
    cfg.MODEL.DEVICE = 'cpu'
    cfg.MODEL.WEIGHTS = 'output/model_0029999.pth'
    cfg.freeze()
    default_setup(cfg, args)
    return cfg

def main(args):
    cfg = setup(args)
    if cfg.SEMISUPNET.Trainer == "ateacher":
        Trainer = ATeacherTrainer
    else:
        raise ValueError("Trainer Name is not found.")

    if cfg.SEMISUPNET.Trainer == "ateacher":
        model = Trainer.build_model(cfg)

        model_teacher = Trainer.build_model(cfg)
        ensem_ts_model = EnsembleTSModel(model_teacher, model)

        DetectionCheckpointer(
            ensem_ts_model, save_dir=cfg.OUTPUT_DIR
        ).load('output/model_0029999.pth')
        finalModel = ensem_ts_model.modelTeacher

    else:
        finalModel = Trainer.build_model(cfg)
        DetectionCheckpointer(finalModel, save_dir=cfg.OUTPUT_DIR).load(
            'output/model_0004999.pth'
        )
    finalModel.eval()
    #predictor = DefaultPredictor(cfg)

    im = cv2.imread("/scratch/project_2005695/PyTorch-CycleGAN/datasets/TLessRendered/JPEGImages/000003_000001.jpg")

    with torch.no_grad():  # https://github.com/sphinx-doc/sphinx/issues/4258
        im = im[:, :, ::-1]
        height, width = im.shape[:2]
        #im = ResizeShortestEdge.get_transform(im).apply_image(im)
        image = torch.as_tensor(im.astype("float32").transpose(2, 0, 1))

        inputs = {"image": image, "height": height, "width": width}
        predictions = finalModel([inputs])[0]

        print(predictions)

        v = Visualizer(im)
        out = v.draw_instance_predictions(predictions["instances"].to("cpu"))
        cv2.imwrite("/scratch/project_2005695/master-thesis-equipment-detection/misc/adaptive_teacher/output/TLessReal_predictions/predictions.jpg", out.get_image()[..., ::-1][..., ::-1])

    im = cv2.imread("/scratch/project_2005695/PyTorch-CycleGAN/datasets/TLessReal/JPEGImages/000004_000001.png")

    with torch.no_grad():  # https://github.com/sphinx-doc/sphinx/issues/4258
        height, width = im.shape[:2]
        image = torch.as_tensor(im.astype("float32").transpose(2, 0, 1))

        inputs = {"image": image, "height": height, "width": width}
        predictions = finalModel([inputs])[0]

        print(predictions)

        v = Visualizer(im)
        out = v.draw_instance_predictions(predictions["instances"].to("cpu"))
        cv2.imwrite("/scratch/project_2005695/master-thesis-equipment-detection/misc/adaptive_teacher/output/TLessReal_predictions/predictions2.jpg", out.get_image()[..., ::-1][..., ::-1])

    return None

if __name__ == "__main__":
    args = default_argument_parser().parse_args()

    print("Command Line Args:", args)
    launch(
        main,
        args.num_gpus,
        num_machines=args.num_machines,
        machine_rank=args.machine_rank,
        dist_url=args.dist_url,
        args=(args,),
    )


import os
import shutil
import sys
import time
from datetime import datetime

# Log to file and console
def log(message, file):
    f = open(file, "a")
    f.write(message + "\n")
    f.close()

    print(message)

def getTime():
    return datetime.now().strftime("%d/%m/%Y %H:%M:%S")

# Sync folders
def sync(src, dest, log_f):
    # Traverse through the source folder
    for root, fldrs, files in os.walk(src):
        dest_path = root.replace(src, dest)

        # Copy files to replica
        for file in files:
            file_src = os.path.join(root, file)
            file_dest = os.path.join(dest_path, file)

            if os.path.exists(file_dest) and os.path.getmtime(file_dest) >= os.path.getmtime(file_src):
                # Destination file exists and is up-to-date, do nothing
                pass
            else:
                shutil.copy(file_src, file_dest)
                log(f"[[ {getTime()} ]] File {file} copied to {file_dest}.", log_f)

        # Copy directories to replica
        for fldr in fldrs:
            try:
                fldr_dest = os.path.join(dest_path, fldr)
                os.mkdir(fldr_dest)
                log(f"[[ {getTime()} ]] Folder {fldr} created in {fldr_dest}.", log_f)
            except OSError:
                pass

    # Traverse through replica folder
    for root, fldrs, files in os.walk(dest):
        src_path = root.replace(dest, src)

        # Delete files no longer present in src folder
        for file in files:
            file_src = os.path.join(src_path, file)
            file_dest = os.path.join(root, file)

            if not os.path.exists(file_src):
                os.remove(file_dest)
                log(f"[[ {getTime()} ]] File {file} deleted from {file_dest}", log_f)

        # Delete folders no longer present in src folder
        for fldr in fldrs:
            fldr_dest = os.path.join(root, fldr)
            fldr_src = os.path.join(src_path, fldr)
            if not os.path.exists(fldr_src):
                shutil.rmtree(fldr_dest)
                log(f"[[ {getTime()} ]] Folder {fldr} deleted from {fldr_dest}", log_f)

# Sync periodically
def syncInterval(src, dest, interval, log_f):
    while True:
        sync(src, dest, log_f)
        time.sleep(int(interval))

syncInterval(sys.argv[1], sys.argv[2], sys.argv[3], sys.argv[4])


from django.views.generic import TemplateView
from web_project import TemplateLayout
from django.shortcuts import render
from django.http import HttpResponse
from scrapy.crawler import CrawlerProcess
from .scraper import MySpider

"""
This file is a view controller for multiple pages as a module.
Here you can override the page view layout.
Refer to sample/urls.py file for more pages.
"""

class SampleView(TemplateView):
    # Predefined function
    def get_context_data(self, **kwargs):
        # A function to init the global layout. It is defined in web_project/__init__.py file
        context = TemplateLayout.init(self, super().get_context_data(**kwargs))

        return context

# views.py

def category_articles(request):
    """
    Scrape articles based on the selected category and render them in a template.

    Args:
        request (HttpRequest): The HTTP request object.

    Returns:
        HttpResponse: The HTTP response object containing the rendered template.
    """
    # Call a function to scrape articles based on the selected category
    process = CrawlerProcess({
        'DOWNLOAD_DELAY': 5,  # Add a delay of 2 seconds between requests
        'CONCURRENT_REQUESTS': 1,  # Limit concurrent requests to 1
        'COOKIES_ENABLED': True,  # Enable cookies for session management
        'RETRY_TIMES': 3,  # Retry failed requests up to 3 times
        'DOWNLOADER_MIDDLEWARES': {
            'scrapy.downloadermiddlewares.retry.RetryMiddleware': 90,
            'scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware': 110,
        },
    })

    process.crawl(MySpider)  # Instantiate MySpider class and crawl the data
    process.start()

    # Process the scraped data to match the format needed for the template
    results = MySpider.results
    articles = []
    for item in results:
        article = {
            'Title': item['Title'],
            'Authors': ', '.join(item['Authors']),  # Convert authors list to a string

        }
        print(article['Title'],article['Authors'])
        articles.append(article)

    # Render the template with the processed data
    return render(request, 'index.html', {'articles': articles})


import csv
import json
import urllib.request as ur

data = []
with open('data.csv', 'r') as f:
  for row in csv.reader(f, delimiter=';'):
    data.append(row)

base_url = "http://maps.google.com/maps/api/geocode/json?address="
for d in data:
  d[2] = d[2].replace(" ","+")
  url = base_url + d[2]
#  print(url)

  response = ur.urlopen(url)
  text = response.read()
  j = json.loads(text)

  if(len(j["results"])):
    with open("out_gmaps.csv", "a") as out:
      lat = j["results"][0]["geometry"]["location"]["lat"]
      lon = j["results"][0]["geometry"]["location"]["lng"]
      s = "%s;%s;%s;%s;%s" % (d[0],d[1],d[2].replace("+"," "),lat,lon)
      out.write(s + "\n")
      print(s)
  else:
    with open("out_gmaps.err", "a") as out:
      s = "%s;%s;%s;%s;%s" % (d[0],d[1],d[2].replace("+"," "),"null","null")
      out.write(s + "\n")
      print(s)


import argparse
from bs4 import BeautifulSoup as soup
from src.parser.video_metadata import VideoMetadata
from src.parser.comment_data import CommentData
import json

class CommentParser(object):

    def __init__(self, source):

        parser = soup(source, "html5lib")

        self.video_metadata = VideoMetadata(parser)

        self.comments = []

        self._parse_comments(parser)

    def _parse_comments(self, parser):

        comment_blocks = parser.find_all('div', id ='main', class_='ytd-comment-renderer')

        for blk in comment_blocks:

            comment = CommentData(blk)
            if comment.author is not None and comment.author != '':
                self.comments.append(comment)

        print(len(self.comments))

    def to_json(self):

        json = {}

        json['metadata'] = self.video_metadata.to_json()

        json['comments'] = []

        for c in self.comments:

            json['comments'].append(c.to_json())

        return json

    def export_json(self, path):

        with open(path, 'w') as flh:
            json.dump(self.to_json(), flh)

    def comments_to_csv(self):

        csv_list = []

        for c in self.comments:

            csv_list.append(c.to_csv())

        return csv_list

    def export_comments_csv(self, path):

        import csv

        with open(path, 'w') as flh:
            csv_list = self.comments_to_csv()
            headers = sorted(list(csv_list[0]))
            writer = csv.DictWriter(flh, fieldnames=headers)
            writer.writeheader()

            writer.writerows(self.comments_to_csv())

if __name__ == '__main__':

    arg_parser = argparse.ArgumentParser(description='Parse html source for video metadata and comments')
    arg_parser.add_argument('--name', type=str, help='name of video page stored in data folder.', required=True)

    args = arg_parser.parse_args()

    with open('../../data/{}_source.html'.format(args.name), 'r') as flh:

        content = flh.read()
        cp = CommentParser(content)
        cp.export_json('../../data/{}_parsed.json'.format(args.name))
        cp.export_comments_csv('../../data/{}_comments.csv'.format(args.name))


from matplotlib.patches import Rectangle
from matplotlib.collections import PatchCollection
from z3support.datamodel.vertex import Vertex
from z3support.datamodel.point import Point
import os
import csv
import matplotlib.pyplot as plt
import matplotlib
import numpy as np

#plt.style.use('aps')
pgf_with_rc_fonts = {"pgf.texsystem": "pdflatex"}
matplotlib.rcParams.update(pgf_with_rc_fonts)

L = 4
LINE_SIZE = 0.05
# link length
LINK_LENGTH = 1.0

# Two types of files
# 1) a file for each configuration; a row for each vertex
# 2) all in one file; a row for each configuration

FILE_TYPE = 1

# 1) z3
# 2) z2
SYSTEM_TYPE = 1

def adjusted_figure():
    fig = plt.figure()
    #fig.subplots_adjust(bottom=0.14,left=0.135,right=0.98,top=0.97)
    fig.subplots_adjust(bottom=0.05, left=0.05, right=0.99, top=0.95)
    ax = fig.add_subplot(111)

    return fig, ax

def draw_lattice_backround(square_lattice_size, ax):
    patches = []

    # horizontal lines
    for i in range(square_lattice_size):
        loc = (-LINK_LENGTH/2.0, float(i) - LINE_SIZE/2.0)
        cur_horz_rect = Rectangle(loc, square_lattice_size, LINE_SIZE, color='k')
        patches.append(cur_horz_rect)
    # vertical lines
    for i in range(square_lattice_size):
        loc = (i - LINE_SIZE/2.0, -LINK_LENGTH/2.0)
        cur_vert_rect = Rectangle(loc, LINE_SIZE, square_lattice_size, color='k')
        patches.append(cur_vert_rect)

    collection = PatchCollection(patches)
    collection.set_color('k')
    ax.add_collection(collection)

def plot_test_points(ax, lat_size):
    rect_patches = []
    tri_patches = []

    location = Point(0, 1)
    vertex = Vertex(lat_size)
    vertex.location = location
    vertex.N = 'Out'
    vertex.E = 'Out'
    vertex.S = 'Out'
    vertex.W = 'Out'
    vertex.make_patches_to_plot(LINK_LENGTH)

    rect_patches += vertex.rect_patches
    tri_patches += vertex.tri_patches

    location = Point(3, 3)
    vertex = Vertex(lat_size)
    vertex.location = location
    vertex.N = 'In'
    vertex.E = 'In'
    vertex.S = 'In'
    vertex.W = 'In'
    vertex.make_patches_to_plot(LINK_LENGTH)

    rect_patches += vertex.rect_patches
    tri_patches += vertex.tri_patches

    collection = PatchCollection(rect_patches)
    collection.set_color('grey')
    ax.add_collection(collection)

    draw_lattice_backround(L, ax)

    collection = PatchCollection(tri_patches)
    collection.set_color('black')
    ax.add_collection(collection)

def list_of_vertices_on_sublattice(l, shaped_array):
    lat_size = Point(l, l)
    vertex_list = []
    for i in range(l):
        for j in range(l):
            # only make a vertex for one sublattice
            if (i + j) % 2 == 0:
                cur_v = Vertex(lat_size)
                cur_row = {}
                if shaped_array[i, j][0] == 1:
                    cur_row['E'] = "Out"
                elif shaped_array[i, j][0] == -1:
                    cur_row['E'] = "Blank"
                if shaped_array[i, j][1] == 1:
                    cur_row['N'] = "Out"
                elif shaped_array[i, j][1] == -1:
                    cur_row['N'] = "Blank"

                if shaped_array[(i-1)%l, j][1] == 1:
                    cur_row['S'] = 'In'
                elif shaped_array[(i-1)%l, j][1] == -1:
                    cur_row['S'] = 'Blank'
                if shaped_array[i, (j-1)%l][0] == 1:
                    cur_row['W'] = 'In'
                elif shaped_array[i, (j-1)%l][0] == -1:
                    cur_row['W'] = 'Blank'

                cur_row['x'] = j
                cur_row['y'] = i
                cur_v.fill_from_csv_row(cur_row)

                vertex_list.append(cur_v)
    return vertex_list

def main():
    lat_size = Point(L, L)

    testing = False

    lattice_files = os.listdir('lattices')

    for cur_f in lattice_files:
        rect_patches = []
        tri_patches = []
        if not (('.csv' in cur_f) or ('.txt' in cur_f)):
            continue

        fig, ax = adjusted_figure()

        if testing:
            plot_test_points(ax, lat_size)
            break

        full_f_name = os.path.join('lattices', cur_f)

        with open(full_f_name, 'r') as csv_file:

            if FILE_TYPE == 1:
                reader = csv.DictReader(csv_file)

                for row in reader:

                    vertex = Vertex(lat_size)
                    vertex.fill_from_csv_row(row)
                    vertex.make_patches_to_plot(LINK_LENGTH, link_width_factor=0.15)

                    rect_patches += vertex.rect_patches
                    tri_patches += vertex.tri_patches
            elif FILE_TYPE == 2:
                if SYSTEM_TYPE == 1:
                    config_array = np.genfromtxt(csv_file, delimiter=',')
                elif SYSTEM_TYPE == 2:
                    config_array = np.genfromtxt(csv_file, delimiter=' ')
                config_array = config_array[0].reshape(L, L, -1)
                vertex_list = list_of_vertices_on_sublattice(L, config_array)

                for cur_vertex in vertex_list:
                    cur_vertex.make_patches_to_plot(LINK_LENGTH, link_width_factor=0.15)
                    rect_patches += cur_vertex.rect_patches
                    if SYSTEM_TYPE == 1:
                        tri_patches += cur_vertex.tri_patches

                print(vertex_list)
            collection = PatchCollection(rect_patches)
            collection.set_color('grey')
            ax.add_collection(collection)

            collection = PatchCollection(tri_patches)
            collection.set_color('black')
            ax.add_collection(collection)

        draw_lattice_backround(L, ax)

        plt.axis('equal')
        plt.axis('off')
        #ax.set_aspect(1.0)

        ax.set_xlim([-0.5, float(L) - 0.5])
        ax.set_ylim([-0.5, float(L) - 0.5])

        #plt.show()
        if not os.path.exists('figures'):
            os.mkdir('figures')
        if not os.path.exists(os.path.join('figures', 'lattices')):
            os.mkdir(os.path.join('figures', 'lattices'))

        full_fig_name = os.path.join('figures', 'lattices', cur_f.split('.')[0] + '.png')
        plt.savefig(full_fig_name, dpi=300)
        plt.close(fig)

    if testing:
        plt.axis('equal')
        plt.show()

if __name__ == "__main__":
    main()


import operator
import os

from param import FeatParams, XPParams, Paths
import logging

class Corpus:
    """
        a class used to encapsulate all the information of the corpus
    """

    mweTokenDic, mweDictionary, mwtDictionary, mwtDictionaryWithSent = {}, {}, {}, {}

    def __init__(self, langName):
        """
            an initializer of the corpus, responsible of creating a structure of objects encapsulating all the information
            of the corpus, its sentences, tokens and MWEs.

            This function iterate over the lines of corpus document to create the precedent ontology
        """
        logging.warn('Language name: {0}'.format(langName))
        self.trainingSents, self.testingSents, self.trainDataSet = [], [], []
        path = os.path.join(Paths.corporaPath, langName)
        mweFile = os.path.join(path, 'train.parsemetsv')
        testMweFile = os.path.join(path, 'test.parsemetsv')
        conlluFile, testConllu = self.getTrainAndTestConlluPath(path)
        testFile = os.path.join(path, 'test.parsemetsv')

        if conlluFile and testConllu:
            self.trainDataSet = Corpus.readConlluFile(conlluFile)
            Corpus.readMweFile(mweFile, self.trainDataSet)

            for sent in self.trainDataSet:
                sent.recognizeEmbedded()
                sent.recognizeInterleavingVMWEs()
                sent.recognizeContinouosandSingleVMWEs()
            if XPParams.realExper:
                self.testDataSet = Corpus.readConlluFile(testConllu)
                Corpus.readMweFile(testMweFile, self.testDataSet)
        else:
            self.trainDataSet = Corpus.readSentences(mweFile)
            self.testDataSet = Corpus.readSentences(testFile, forTest=True)
            for sent in self.trainDataSet:
                sent.recognizeEmbedded()
                sent.recognizeInterleavingVMWEs()
                sent.recognizeContinouosandSingleVMWEs()
        self.getTrainAndTestSents()
        if XPParams.useCrossValidation:
            self.testRange, self.trainRange = self.getRangs()
        else:
            self.testRange, self.trainRange = None, None

    def getTrainAndTestConlluPath(self, path):
        conlluFile, testConllu = None, None
        if XPParams.useAutoGeneratedPOS and XPParams.useAutoGeneratedDEP and os.path.isfile(
                os.path.join(path, 'train.conllu.autoPOS.autoDep')):
            conlluFile = os.path.join(path, 'train.conllu.autoPOS.autoDep')
            if os.path.isfile(os.path.join(path, 'test.conllu.autoPOS.autoDep')):
                testConllu = os.path.join(path, 'test.conllu.autoPOS.autoDep')
        elif XPParams.useAutoGeneratedPOS and os.path.isfile(os.path.join(path, 'train.conllu.autoPOS')):
            conlluFile = os.path.join(path, 'train.conllu.autoPOS')
            if os.path.isfile(os.path.join(path, 'test.conllu.autoPOS')):
                testConllu = os.path.join(path, 'test.conllu.autoPOS')
        elif os.path.isfile(os.path.join(path, 'train.conllu')):
            conlluFile = os.path.join(path, 'train.conllu')
            if os.path.isfile(os.path.join(path, 'test.conllu')):
                testConllu = os.path.join(path, 'test.conllu')
        return conlluFile, testConllu

    @staticmethod
    def readConlluFile(conlluFile):
        sentences = []
        with open(conlluFile) as corpusFile:
            # Read the corpus file
            lines = corpusFile.readlines()
            sent = None
            senIdx = 0
            sentId = ''

            lineNum = 0
            missedUnTag = 0
            missedExTag = 0

            for line in lines:
                if len(line) > 0 and line.endswith('\n'):
                    line = line[:-1]
                if line.startswith('# sentid:'):
                    sentId = line.split('# sentid:')[1].strip()

                elif line.startswith('# sentence-text:'):
                    continue

                elif line.startswith('1\t'):
                    if sentId.strip() != '':
                        sent = Sentence(senIdx, sentid=sentId)
                    else:
                        sent = Sentence(senIdx)
                    senIdx += 1
                    sentences.append(sent)

                if not line.startswith('#'):
                    lineParts = line.split('\t')

                    if len(lineParts) != 10 or '-' in lineParts[0]:
                        continue

                    lineNum += 1
                    if lineParts[3] == '_':
                        missedUnTag += 1
                    if lineParts[4] == '_':
                        missedExTag += 1

                    morpho = ''
                    if lineParts[5] != '_':
                        morpho = lineParts[5].split('|')
                    if lineParts[6] != '_':
                        token = Token(lineParts[0], lineParts[1].lower(), lemma=lineParts[2],
                                      abstractPosTag=lineParts[3], morphologicalInfo=morpho,
                                      dependencyParent=int(lineParts[6]),
                                      dependencyLabel=lineParts[7])
                    else:
                        token = Token(lineParts[0], lineParts[1].lower(), lemma=lineParts[2],
                                      abstractPosTag=lineParts[3], morphologicalInfo=morpho,
                                      dependencyLabel=lineParts[7])
                    if XPParams.useUniversalPOSTag:
                        token.posTag = lineParts[3]
                    else:
                        if lineParts[4] != '_':
                            token.posTag = lineParts[4]
                        else:
                            token.posTag = lineParts[3]
                    # Associate the token with the sentence
                    sent.tokens.append(token)
                    sent.text += token.text + ' '
        return sentences

    @staticmethod
    def readMweFile(mweFile, sentences):
        mweNum = 0
        with open(mweFile) as corpusFile:

            # Read the corpus file
            lines = corpusFile.readlines()
            noSentToAssign = False
            sentIdx = 0
            for line in lines:
                if line == '\n' or line.startswith('# sentence-text:') or (
                        line.startswith('# sentid:') and noSentToAssign):
                    continue
                if len(line) > 0 and line.endswith('\n'):
                    line = line[:-1]
                if line.startswith('1\t'):
                    sent = sentences[sentIdx]
                    sentIdx += 1
                lineParts = line.split('\t')
                if '-' in lineParts[0]:
                    continue
                if lineParts is not None and len(lineParts) == 4 and lineParts[3] != '_':

                    token = sent.tokens[int(lineParts[0]) - 1]
                    vMWEids = lineParts[3].split(';')
                    for vMWEid in vMWEids:
                        id = int(vMWEid.split(':')[0])
                        # New MWE captured
                        if id not in sent.getWMWEIds():
                            if len(vMWEid.split(':')) > 1:
                                type = str(vMWEid.split(':')[1])
                                vMWE = VMWE(id, token, type)
                            else:
                                vMWE = VMWE(id, token)
                            mweNum += 1
                            sent.vMWEs.append(vMWE)
                        # Another token of an under-processing MWE
                        else:
                            vMWE = sent.getVMWE(id)
                            if vMWE is not None:
                                vMWE.addToken(token)
                        # associate the token with the MWE
                        token.setParent(vMWE)
        return mweNum

    @staticmethod
    def readSentences(mweFile, forTest=False):
        sentences = []
        sentNum, mweNum = 0, 0
        with open(mweFile) as corpusFile:
            # Read the corpus file
            lines = corpusFile.readlines()
            sent = None
            senIdx = 1
            for line in lines:
                if len(line) > 0 and line.endswith('\n'):
                    line = line[:-1]
                if line.startswith('1\t'):
                    # sentId = line.split('# sentid:')[1]
                    if sent is not None:
                        # Represent the sentence as a sequece of tokens and POS tags
                        sent.setTextandPOS()
                        # if not forTest:
                        sent.recognizeEmbedded()
                        sent.recognizeInterleavingVMWEs()

                    sent = Sentence(senIdx)
                    senIdx += 1
                    sentences.append(sent)

                elif line.startswith('# sentence-text:'):
                    if len(line.split(':')) > 1:
                        sent.text = line.split('# sentence-text:')[1]

                lineParts = line.split('\t')

                # Empty line or lines of the form: "8-9	can't	_	_"
                if len(lineParts) != 4 or '-' in lineParts[0]:
                    continue
                token = Token(lineParts[0], lineParts[1])
                # Trait the MWE
                # if not forTest and lineParts[3] != '_':
                if lineParts[3] != '_':
                    vMWEids = lineParts[3].split(';')
                    for vMWEid in vMWEids:
                        id = int(vMWEid.split(':')[0])
                        # New MWE captured
                        if id not in sent.getWMWEIds():
                            type = str(vMWEid.split(':')[1])
                            vMWE = VMWE(id, token, type)
                            mweNum += 1
                            sent.vMWEs.append(vMWE)
                        # Another token of an under-processing MWE
                        else:
                            vMWE = sent.getVMWE(id)
                            if vMWE is not None:
                                vMWE.addToken(token)
                        # associate the token with the MWE
                        token.setParent(vMWE)
                # Associate the token with the sentence
                sent.tokens.append(token)
            sentNum = len(sentences)
            return sentences

    @staticmethod
    def getMWEDic(sents):
        mweDictionary, mweTokenDictionary, mwtDictionary = {}, {}, {}
        for sent in sents:
            for mwe in sent.vMWEs:
                lemmaString = mwe.getLemmaString()
                if len(mwe.tokens) == 1:
                    if lemmaString not in mwtDictionary:
                        mwtDictionary[lemmaString] = mwe.type
                    if lemmaString not in Corpus.mwtDictionaryWithSent:
                        Corpus.mwtDictionaryWithSent[lemmaString] = [sent]
                    elif lemmaString in Corpus.mwtDictionaryWithSent and Corpus.mwtDictionaryWithSent[
                        lemmaString] is not None:
                        Corpus.mwtDictionaryWithSent[lemmaString] = Corpus.mwtDictionaryWithSent[lemmaString].append(
                            sent)
                if lemmaString in mweDictionary:
                    mweDictionary[lemmaString] += 1
                    for token in mwe.tokens:
                        if token.lemma.strip() != '':
                            mweTokenDictionary[token.lemma] = 1
                        else:
                            mweTokenDictionary[token.text] = 1
                else:
                    mweDictionary[lemmaString] = 1
                    for token in mwe.tokens:
                        if token.lemma.strip() != '':
                            mweTokenDictionary[token.lemma] = 1
                        else:
                            mweTokenDictionary[token.text] = 1
        if FeatParams.usePreciseDictionary:
            for key1 in mweDictionary.keys():
                for key2 in mweDictionary.keys():
                    if key1 != key2:
                        if key1 in key2:
                            mweDictionary.pop(key1, None)
                        elif key2 in key1:
                            mweDictionary.pop(key2, None)
        return mweDictionary, mweTokenDictionary, mwtDictionary

    def initializeSents(self, training=True):
        # Erasing each effect of the previous iteration
        sents = self.trainingSents
        if not training:
            sents = self.testingSents

        for sent in sents:
            sent.identifiedVMWEs = []
            sent.initialTransition = None
            sent.featuresInfo = []
            sent.blackMergeNum = 0
            for mwe in sent.vMWEs:
                mwe.isInTrainingCorpus = 0

    def getTrainAndTestSents(self):

        if XPParams.debug:
            self.trainingSents = self.trainDataSet[:500]
            self.testingSents = self.testDataSet[:200]
        elif XPParams.realExper:
            self.trainingSents = self.trainDataSet
            self.testingSents = self.testDataSet

        elif XPParams.useCrossValidation:
            self.trainDataSet = self.trainDataSet
            self.testDataSet = []
            return [], []

        if len(self.trainingSents) <= 0:
            idx = 0
            self.trainingSents, self.testingSents = [], []
            for sent in self.trainDataSet:
                if idx % 5 == 0:
                    self.testingSents.append(sent)
                else:
                    self.trainingSents.append(sent)
                idx += 1

        return self.trainingSents, self.testingSents

    def getRangs(self):
        sents = self.trainDataSet
        testNum = int(len(sents) * 0.2)
        testRanges = [[0, testNum], [testNum, 2 * testNum], [2 * testNum, 3 * testNum], [3 * testNum, 4 * testNum],
                      [4 * testNum, len(sents)]]

        trainRanges = [[testNum, len(sents)], [0, testNum, 2 * testNum, len(sents)],
                       [0, 2 * testNum, 3 * testNum, len(sents)], [0, 3 * testNum, 4 * testNum, len(sents)],
                       [0, 4 * testNum]]

        return testRanges, trainRanges

    def divideSents(self):
        x = XPParams.currentIteration
        if self.testRange is None or self.trainRange is None:
            return
        self.testingSents = self.trainDataSet[self.testRange[x][0]: self.testRange[x][1]]
        if len(self.trainRange[x]) == 2:
            self.trainingSents = self.trainDataSet[self.trainRange[x][0]: self.trainRange[x][1]]
        else:
            self.trainingSents = self.trainDataSet[self.trainRange[x][0]: self.trainRange[x][1]] + \
                                 self.trainDataSet[self.trainRange[x][2]: self.trainRange[x][3]]

    def update(self):
        if XPParams.useCrossValidation:
            self.divideSents()
            self.initializeSents()
        Corpus.mweDictionary, Corpus.mweTokenDic, Corpus.mwtDictionary = Corpus.getMWEDic(self.trainingSents)

    @staticmethod
    def getNewIdentifiedMWE(testingSents):

        idenMWEs = 0
        newIdenMWEs = 0
        semiNewIdenMWEs = 0
        for sent in testingSents:
            for mwe in sent.vMWEs:
                if mwe.getLemmaString() not in Corpus.mweDictionary.keys():
                    idenMWEs += 1
            for mwe in sent.identifiedVMWEs:
                if mwe.getLemmaString() not in Corpus.mweDictionary.keys():
                    for vmw1 in sent.vMWEs:
                        if vmw1.getLemmaString() == mwe.getLemmaString():
                            newIdenMWEs += 1
                            break
                elif mwe.getLemmaString() in Corpus.mweDictionary.keys() and \
                        Corpus.mweDictionary[mwe.getLemmaString()] < 5:
                    semiNewIdenMWEs += 1

        return float(newIdenMWEs) / idenMWEs, float(semiNewIdenMWEs) / idenMWEs

    def __iter__(self):
        for sent in self.trainingSents:
            yield sent

    def __str__(self):
        res = ''
        for sent in self.testingSents:
            tokenList = []
            for token in sent.tokens:
                tokenList.append(token.text.strip())
            labels = ['_'] * len(tokenList)
            for mwe in sent.identifiedVMWEs:
                for token in mwe.tokens:
                    if labels[token.position - 1] == '_':
                        labels[token.position - 1] = str(mwe.id)
                    else:
                        labels[token.position - 1] += ';' + str(mwe.id)
                    if mwe.tokens[0] == token and mwe.type:
                        labels[token.position - 1] += ':' + mwe.type

            for i in range(len(tokenList)):
                res += '{0}\t{1}\t{2}\t{3}\n'.format(i + 1, tokenList[i], '_', labels[i])
            res += '\n'
        return res

class Sentence:
    """
       a class used to encapsulate all the information of a sentence
    """

    def __init__(self, id, sentid=''):

        self.sentid = sentid
        self.id = id
        self.tokens = []
        self.vMWEs = []
        self.identifiedVMWEs = []
        self.text = ''
        self.initialTransition = None
        self.featuresInfo = []
        self.containsEmbedding = False
        self.containsInterleaving = False
        self.containsDistributedEmbedding = False
        self.withRandomSelection = False
        self.blackMergeNum, self.interleavingNum, self.embeddedNum, self.distributedEmbeddingNum = 0, 0, 0, 0

    def getWMWEs(self):
        return self.vMWEs

    def getWMWEIds(self):
        result = []
        for vMWE in self.vMWEs:
            result.append(vMWE.getId())
        return result

    def getVMWE(self, id):

        for vMWE in self.vMWEs:
            if vMWE.getId() == int(id):
                return vMWE
        return None

    def setTextandPOS(self):

        tokensTextList = []
        for token in self.tokens:
            self.text += token.text + ' '
            tokensTextList.append(token.text)
        self.text = self.text.strip()

    def recognizeEmbedded(self, recognizeIdentified=False):
        if recognizeIdentified:
            vmws = self.identifiedVMWEs
        else:
            vmws = self.vMWEs

        if len(vmws) <= 1:
            return 0
        result = 0
        # [x1; x2; x3
        for vMwe1 in vmws:
            if vMwe1.isEmbedded:
                continue
            for vMwe2 in vmws:
                if vMwe1 is not vMwe2 and len(vMwe1.tokens) < len(vMwe2.tokens):
                    if vMwe1.getString() in vMwe2.getString():
                        vMwe1.isEmbedded = True
                        if not recognizeIdentified:
                            self.embeddedNum += 1
                            self.containsEmbedding = True
                        result += 1
                    else:
                        isEmbedded = True
                        vMwe2Lemma = vMwe2.getLemmaString()
                        for token in vMwe1.tokens:
                            if token.getLemma() not in vMwe2Lemma:
                                isEmbedded = False
                                break
                        if isEmbedded:
                            vMwe1.isDistributedEmbedding = True
                            vMwe1.isEmbedded = True
                            if not recognizeIdentified:
                                self.containsDistributedEmbedding = True
                                self.embeddedNum += 1
                                self.distributedEmbeddingNum += 1
                                self.containsEmbedding = True
                            result += 1
        if not recognizeIdentified:
            self.getDirectParents()
        return result

    def recognizeContinouosandSingleVMWEs(self):
        singleWordExp, continousExp = 0, 0
        for mwe in self.vMWEs:
            if len(mwe.tokens) == 1:
                mwe.isSingleWordExp = True
                mwe.isContinousExp = True
                singleWordExp += 1
                continousExp += 1
            else:
                if self.isContinousMwe(mwe):
                    continousExp += 1
        return singleWordExp, continousExp

    def isContinousMwe(self, mwe):
        idxs = []
        for token in mwe.tokens:
            idxs.append(self.tokens.index(token))
        range = xrange(min(idxs), max(idxs))
        mwe.isContinousExp = True
        for i in range:
            if i not in idxs:
                mwe.isContinousExp = False
        return mwe.isContinousExp

    def recognizeInterleavingVMWEs(self):
        if len(self.vMWEs) <= 1:
            return 0
        result = 0
        for vmwe in self.vMWEs:
            if vmwe.isEmbedded or vmwe.isInterleaving:
                continue
            for token in vmwe.tokens:
                if len(token.parentMWEs) > 1:
                    for parent in token.parentMWEs:
                        if parent is not vmwe:
                            if parent.isEmbedded or parent.isInterleaving:
                                continue
                            if len(parent.tokens) <= len(vmwe.tokens):
                                parent.isInterleaving = True
                            else:
                                vmwe.isInterleaving = True
                            self.containsInterleaving = True
                            self.interleavingNum += 1
                            result += 1
        return result

    def getCorpusText(self, gold=True):
        if gold:
            mwes = self.vMWEs
        else:
            mwes = self.identifiedVMWEs
        lines = ''
        idx = 1
        for token in self.tokens:
            line = str(idx) + '\t' + token.text + '\t_\t'
            idx += 1
            for mwe in mwes:
                if token in mwe.tokens:
                    if line.endswith('\t'):
                        line += str(mwe.id)
                    else:
                        line += ';' + str(mwe.id)
                    if token == mwe.tokens[0]:
                        line += ':' + str(mwe.type)
            if line.endswith('\t'):
                line += '_'
            lines += line + '\n'
        return lines

    def getCorpusTextWithPlus(self):
        goldMwes = self.vMWEs
        predMwes = self.identifiedVMWEs
        lines = ''
        idx = 1
        for token in self.tokens:
            line = str(idx) + '\t' + token.text + '\t_\t'
            idx += 1
            for mwe in goldMwes:
                if token in mwe.tokens:
                    if line.endswith('\t'):
                        line += '+'
                        break

            if line.endswith('\t'):
                line += '_\t'
            else:
                line += '\t'
            for mwe in predMwes:
                if token in mwe.tokens:
                    if line.endswith('\t'):
                        line += '+'
                        break
            if line.endswith('\t'):
                line += '_'
            lines += line + '\n'
        return lines

    def isPrintable(self):
        if len(self.vMWEs) > 2:
            return True
        # if not PrintParams.printSentsWithEmbeddedMWEs:
        #     return
        # for mwe in self.vMWEs:
        #     if mwe.isEmbedded:
        #         return True
        return False

        # if PrintParams.printSentsWithEmbeddedMWEs and len(self.vMWEs) > 2:
        #     for mwe in self.vMWEs:
        #         if mwe.isEmbedded:
        #             return True
        # return False

    def getDirectParents(self):

        for token in self.tokens:
            token.getDirectParent()

    @staticmethod
    def getTokens(elemlist):
        if isinstance(elemlist, Token):
            return [elemlist]
        if isinstance(elemlist, list):
            result = []
            for elem in elemlist:
                if isinstance(elem, Token):
                    result.append(elem)
                elif isinstance(elem, list):
                    result.extend(Sentence.getTokens(elem))
            return result
        return [elemlist]

    @staticmethod
    def getTokenLemmas(tokens):
        text = ''
        tokens = Sentence.getTokens(tokens)
        for token in tokens:
            if token.lemma != '':
                text += token.lemma + ' '
            else:
                text += token.text + ' '
        return text.strip()

    def printSummary(self):
        vMWEText = ''
        for vMWE in self.vMWEs:
            vMWEText += str(vMWE) + '\n'
        if len(self.identifiedVMWEs) > 0:
            identifiedMWE = '### Identified MWEs: \n'
            for mwe in self.identifiedVMWEs:
                identifiedMWE += str(mwe) + '\n'
        else:
            identifiedMWE = ''

        return '## Sentence No. ' + str(self.id) + ' - ' + self.sentid + '\n' + self.text + \
               '\n### Existing MWEs: \n' + vMWEText + identifiedMWE

    def __str__(self):

        vMWEText = ''
        for vMWE in self.vMWEs:
            vMWEText += str(vMWE) + '\n\n'
        if len(self.identifiedVMWEs) > 0:
            identifiedMWE = '### Identified MWEs: \n'
            for mwe in self.identifiedVMWEs:
                identifiedMWE += str(mwe) + '\n\n'
        else:
            identifiedMWE = ''
        # featuresInfo = ''

        result = ''
        transition = self.initialTransition
        idx = 0
        tab = ' '
        while True:
            if transition is not None:
                if transition.type is not None:
                    type = transition.type.name
                else:
                    type = tab * 8
                configuration = str(transition.configuration)
                if type.startswith('MERGE') or type.startswith('WHITE'):
                    type = '**' + type + '**' + tab * 3
                if len(type) == 'SHIFT':
                    type = type + tab * 3
                result += '\n\n' + str(
                    transition.id) + '- ' + type + tab * 3 + '>' + tab * 3 + configuration + '\n\n'
                if transition.next is None:
                    break
                transition = transition.next
                idx += 1
            else:  # result += str(self.featuresInfo[1][idx]) + '\n\n'
                break
        text = ''
        for token in self.tokens:
            if token.parentMWEs is not None and len(token.parentMWEs) > 0:
                text += '**' + token.text + '**' + ' '
            else:
                text += token.text + ' '

        return '## Sentence No. ' + str(self.id) + ' - ' + self.sentid + '\n' + text + \
               '\n### Existing MWEs: \n' + vMWEText + identifiedMWE  # + 'black Merge Num : ' + str(self.blackMergeNum) + ' Interleaving Num: ' + str(self.interleavingNum) \
        # + '\n' + result #+ str(self.initialTransition) + '\n### Features: \n' + featuresInfo

    def __iter__(self):
        for vmwe in self.vMWEs:
            yield vmwe

class VMWE:
    """
        A class used to encapsulate the information of a verbal multi-word expression
    """

    def __init__(self, id, token=None, type='', isEmbedded=False, isInterleaving=False, isDistributedEmbedding=False,
                 isInTrainingCorpus=0):
        self.id = int(id)
        self.isInTrainingCorpus = isInTrainingCorpus
        self.tokens = []
        self.isSingleWordExp = False
        self.isContinousExp = False
        if token is not None:
            self.tokens.append(token)
        self.type = type
        self.isEmbedded = isEmbedded
        self.isDistributedEmbedding = isDistributedEmbedding
        self.isInterleaving = isInterleaving
        self.isVerbal = True
        self.directParent = None

    def getId(self):
        return self.id

    def addToken(self, token):
        self.tokens.append(token)

    @staticmethod
    def getVMWENumber(tokens):
        result = 0
        for token in tokens:
            if isinstance(token, VMWE):
                result += 1
        return result

    @staticmethod
    def haveSameParents(tokens):
        # Do they have all a parent?
        for token in tokens:
            if not token.parentMWEs:
                return None
        # Get all parents of tokens
        parents = set()
        for token in tokens:
            for parent in token.parentMWEs:
                parents.add(parent)
        if len(parents) == 1:
            return list(parents)

        selectedParents = list(parents)
        for parent in parents:
            for token in tokens:
                if parent not in token.parentMWEs:
                    if parent in selectedParents:
                        selectedParents.remove(parent)

        for parent in list(selectedParents):
            if parent.isInterleaving or parent.isDistributedEmbedding:
                selectedParents.remove(parent)
        return selectedParents

    # @staticmethod
    # def haveSameDirectParents(s0, s1):
    #     if isinstance(s0, Token) and isinstance(s1, Token):
    #         return s0.directParent == s1.directParent
    # if isinstance(s0, list):

    @staticmethod
    def getParents(tokens, type=None):
        if len(tokens) == 1:
            if tokens[0].parentMWEs:
                for vmwe in tokens[0].parentMWEs:
                    if len(vmwe.tokens) == 1:  # and vmwe.type.lower() != type:
                        if type is not None:
                            if vmwe.type.lower() == type.lower():
                                return [vmwe]
                            else:
                                return None
                        else:
                            return [vmwe]

        # Do they have all a parent?
        for token in tokens:
            if len(token.parentMWEs) == 0:
                return None

        # Get all parents of tokens
        parents = set()
        for token in tokens:
            for parent in token.parentMWEs:
                parents.add(parent)
        selectedParents = list(parents)
        for parent in parents:
            if len(parent.tokens) != len(tokens):
                if parent in selectedParents:
                    selectedParents.remove(parent)
                continue
            for token in tokens:
                if parent not in token.parentMWEs:
                    if parent in selectedParents:
                        selectedParents.remove(parent)
        for parent in list(selectedParents):
            if parent.isInterleaving or parent.isDistributedEmbedding:
                selectedParents.remove(parent)
        if type is not None:
            for parent in list(selectedParents):
                if parent.type.lower() != type:
                    selectedParents.remove(parent)
        return selectedParents

    # def getDirectParent(self):
    #     self.directParent = None
    #     if self.parentMWEs is not None and len(self.parentMWEs)> 0:
    #         if len(self.parentMWEs) == 1:
    #             self.directParent = self.parentMWEs[0]
    #         else:
    #             parents = sorted(self.parentMWEs, key=lambda VMWE : (VMWE.isInterleaving, VMWE.isEmbedded, len(VMWE.tokens) ))
    #             for parent in parents:
    #                 if not parent.isInterleaving:
    #                     self.directParent = parent
    #                     break
    #     return self.directParent

    def __str__(self):
        tokensStr = ''
        for token in self.tokens:
            tokensStr += token.text + ' '
        tokensStr = tokensStr.strip()
        isInterleaving = ''
        if self.isInterleaving:
            isInterleaving = ', Interleaving '
        isEmbedded = ''
        if self.isEmbedded:
            if self.isDistributedEmbedding:
                isEmbedded = ', DistributedEmbedding '
            else:
                isEmbedded = ', Embedded '
            # isContinousExp =''
            # if self.isContinousExp:
            # isContinousExp = 'Continous'
        type = ''
        if self.type != '':
            type = '(' + self.type
            if self.isInTrainingCorpus != 0:
                type += ', ' + str(self.isInTrainingCorpus) + ')'
            else:
                type += ')'
        return str(self.id) + '- ' + '**' + tokensStr + '** ' + type + isEmbedded + isInterleaving

    def __iter__(self):
        for t in self.tokens:
            yield t

    def getString(self):
        result = ''
        for token in self.tokens:
            result += token.text + ' '
        return result[:-1].lower()

    def getLemmaString(self):
        result = ''
        for token in self.tokens:
            if token.lemma.strip() != '':
                result += token.lemma + ' '
            else:
                result += token.text + ' '
        return result[:-1].lower()

    def In(self, vmwes):

        for vmwe in vmwes:
            if vmwe.getString() == self.getString():
                return True

        return False

    def __eq__(self, other):
        if not isinstance(other, VMWE):
            raise TypeError()
        if self.getLemmaString() == other.getLemmaString():
            return True
        return False

    def __hash__(self):
        return hash(self.getLemmaString())

    def __contains__(self, vmwe):
        if not isinstance(vmwe, VMWE):
            raise TypeError()
        if vmwe is self or vmwe.getLemmaString() == self.getLemmaString():
            return False
        if vmwe.getLemmaString() in self.getLemmaString():
            return True
        for token in vmwe.tokens:
            if token.getLemma() not in self.getLemmaString():
                return False
        return True

class Token:
    """
        a class used to encapsulate all the information of a sentence tokens
    """

    def __init__(self, position, txt, lemma='', posTag='', abstractPosTag='', morphologicalInfo=[], dependencyParent=-1,
                 dependencyLabel=''):
        self.position = int(position)
        self.text = txt
        self.lemma = lemma
        self.abstractPosTag = abstractPosTag
        self.posTag = posTag
        self.morphologicalInfo = morphologicalInfo
        self.dependencyParent = dependencyParent
        self.dependencyLabel = dependencyLabel
        self.parentMWEs = []
        self.directParent = None

    def setParent(self, vMWE):
        self.parentMWEs.append(vMWE)

    def getLemma(self):
        if self.lemma != '':
            return self.lemma.strip()
        return self.text.strip()

    def getDirectParent(self):
        self.directParent = None
        if self.parentMWEs is not None and len(self.parentMWEs) > 0:
            if len(self.parentMWEs) == 1:
                if not self.parentMWEs[0].isInterleaving:
                    self.directParent = self.parentMWEs[0]
            else:
                parents = sorted(self.parentMWEs,
                                 key=lambda VMWE: (VMWE.isInterleaving, VMWE.isEmbedded, len(VMWE.tokens)),
                                 reverse=True)
                for parent in parents:
                    if not parent.isInterleaving:
                        self.directParent = parent
                        break
        return self.directParent

    def In(self, vmwe):
        for token in vmwe.tokens:
            if token.text.lower() == self.text.lower() and token.position == self.position:
                return True
        return False

    def isMWT(self):
        if self.parentMWEs:
            for vmw in self.parentMWEs:
                if len(vmw.tokens) == 1:
                    return vmw
        return None

    def __str__(self):
        parentTxt = ''
        if len(self.parentMWEs) != 0:
            for parent in self.parentMWEs:
                parentTxt += str(parent) + '\n'

        return str(self.position) + ' : ' + self.text + ' : ' + self.posTag + '\n' + 'parent VMWEs\n' + parentTxt

def getTokens(elemlist):
    if str(elemlist.__class__) == 'corpus.Token':  # isinstance(elemlist, Token):
        return [elemlist]
    if isinstance(elemlist, list):
        result = []
        for elem in elemlist:
            if str(elem.__class__) == 'corpus.Token':
                result.append(elem)
            elif isinstance(elem, list) and len(elem) == 1 and isinstance(elem[0], list):
                result.extend(getTokens(elem[0]))
            elif isinstance(elem, list) and len(elem):
                result.extend(getTokens(elem))
        return result
    return [elemlist]


import pandas as pd
import dash
from dash import html, dcc, callback, dash_table, Input, Output, State
import dash_bootstrap_components as dbc
import plots
import data

dash.register_page(__name__, path='/explore')

df = data.read_json_data('data/biosignature.json')
df = pd.DataFrame(df)

def generate_data_preview(df):
    return df.to_dict('records'),[{'id': x, 'name': x, 'presentation': 'markdown'} if x == 'pub_url' else {'id': x, 'name': x} for x in df.columns], [dict(selector='td[data-dash-column="pub_url"] table', rule='color: blue;')]

def generate_data_to_validate(df):
    df = df[df['status'] == 'ðŸŸ  pending']
    return df.to_dict('records'), [{'id': x, 'name': x, 'presentation': 'markdown'} if x == 'pub_url' else {'id': x, 'name': x} for x in df.columns], [dict(selector='td[data-dash-row="1"][data-dash-column="pub_url"] table', rule='color: blue;')]

def create_dropdown(df):
    df = df[df['status'] == ' ðŸŸ¢ validated']
    options = df['paleoenvironment'].unique().tolist()
    options = ['All'] + options
    return dcc.Dropdown(id = 'paleoenv-value', options = options, value = 'All', clearable= False,
                        placeholder = 'Paleoenvironment', style = {'color': 'black', 'width': '30vw'})

layout = html.Div(children=[
        dbc.Card(
            [
                dbc.CardHeader(
                    dbc.Tabs(
                        [
                            dbc.Tab(label="Dashboard", tab_id="dashboard"),
                            dbc.Tab(label="Data", tab_id="data"),
                        ],
                        style = {'margin-top': '0.5vh', 'margin-left': '0.1vw'},
                        id="card-tabs",
                        active_tab="dashboard",
                    ), style= {'z-index': '1000'}
                ),
                dbc.CardBody(html.P(id="card-content", className="card-text")),
            ], style = {'z-index': 0}),

    ], style={'z-index': '1'})

@callback(Output('card-content', 'children'),
          Input('card-tabs', 'active_tab'))
def render_tab_content(tab_value):
    #df = data.read_database()
    if tab_value == 'dashboard':
        return html.Div([
            html.Div([
                html.Div(id='start-page', children=[
                    html.Div([
                        html.Div(children = [
                            html.Div([
                                html.H3('Projection', style = {'order':'1', 'text-align': 'center', 'color': 'black', 'margin-bottom': '0px', 'font-family': 'Arial, sans-serif', 'font-size': '1.5vw'}),
                                dcc.RadioItems(id = 'projection-selector',
                                            options = ['mercator', 'orthographic'],
                                            value = 'mercator', inline = True, style = {'order': '2', 'color': 'black', 'width': '30vw'})
                                ], className = 'projection-selector')], className= 'div-projection'),
                            html.Div([
                                html.H3('(Paleo)environment', style = {'order':'1', 'text-align': 'left', 'color': 'black', 'margin-bottom': '1vh', 'font-family': 'Arial, sans-serif', 'font-size': '1.5vw'}),
                                html.Div(id = 'paleoenv-filter', children = [create_dropdown(df)], style = {'order': '2', 'z-index': '10', 'position': 'absolute'})
                            ], style = {'margin-top': '1vh'}),
                            html.Div(id = 'interactive-map', children = [], className = 'interactive-map')],
                            style = {'order': '1', 'margin-left': '0vw', 'margin-top': '2vh'}),

                    html.Div([
                        html.Div(id = 'hover-bar-chart', children = [], style = {'order': '1'}),
                        html.Div(id = 'hover-pie-chart', children = [], style = {'order': '2'})],
                        className= 'hover-charts')

                ], className = 'main-panel'),

            html.Div([
                html.Div([
                    html.H3('Mars counterpart(s)', style = {'color': 'black', 'text-align': 'center', 'margin-bottom': '-2vh', 'margin-left': '4vw', 'order': '1', 'font-family': 'Arial, sans-serif', 'font-size': '1.5vw'}),
                    html.Div(id = 'mars-map', children = [],
                            style = {'width': '30vw', 'height': '25vh', 'order': '2', 'margin-left': '2vw', 'margin-top': '3vh'})],
                style = {'order':'1','display':'flex', 'flex-direction': 'column', 'align-items': 'center','margin-top': '0vh', 'height': '25vh'}),
                html.Div([
                    html.H3('Data preview', style = {'color': 'black', 'text-align': 'center', 'order':'1', 'font-family': 'Arial, sans-serif', 'font-size': '1.5vw'}),
                    html.Div([
                        dash_table.DataTable(id = 'data-preview', data = generate_data_preview(df)[0], columns= generate_data_preview(df)[1],
                                            css = generate_data_preview(df)[2] ,editable = False, style_data = {'color': 'black', 'font-family': 'Arial, sans serif'},
                                            style_header= {'color': 'black', 'font-weight': 'bold', 'background-color': 'rgba(5, 8, 184, 0.4)', 'textAlign': 'left', 'font-family': 'Arial, sans serif'},
                                            style_cell = {'textAlign': 'left', 'padding': '0px'},
                                            style_table = {'width':'30vw', 'height':'37.5vh', 'overflow': 'auto'},
                                            style_data_conditional=[
                                                                        {
                                                                            'if': {'row_index': 'even'},
                                                                            'backgroundColor': 'rgba(5, 8, 184, 0.1)',
                                                                        },
                                                                        {
                                                                            'if': {'row_index': 'odd'},
                                                                            'backgroundColor': 'rgba(5, 8, 184, 0.4)',
                                                                        },
                                                                        {
                                                                            'if': {'column_id': 'pub_url'},
                                                                            'color': 'black',
                                                                            'padding-top': '17px'
                                                                        }
                                                                    ])],
                        style = {'order':'2', 'height': '38vh'})],
                style = {'order':'2','display':'flex', 'flex-direction': 'column', 'align-items': 'center',
                        'width':'30vw', 'height':'45vh', 'margin-top': '7vh', 'margin-left': '1vw'})],
                className = 'left-panel')],

            className = 'main-body')], style = {'height': '80vh','margin':'auto', 'margin-top': '-4.5vh', 'width': '98vw', 'backgroundColor': '#e6e6e6', 'border-radius': '10px'})
    else:
        return html.Div([
                    html.Div([
                        html.P(id='datatable-message', style = {'order':'1', 'color':'black', 'font-family': 'Arial, sans-serif', 'font-size': '1.3vw', 'margin-top': '1vh'}),

                        html.Div(id='data-tab-buttons', style = {'display': 'flex', 'flex-direction': 'row', 'align-items': 'center', 'justify-content': 'flex-end', 'order': '2', 'width': '40vw', 'margin-top':'0.5vh'}),

                    ], style={'width': '95vw', 'margin': 'auto', 'margin-top': '1vh', 'display': 'flex', 'flex-direction': 'row', 'align-items': 'center', 'justify-content': 'space-between'}),
                    dash_table.DataTable(id = 'data-preview', data = generate_data_preview(df)[0], columns= generate_data_preview(df)[1], export_format = 'csv',
                                css = generate_data_preview(df)[2], editable = False, style_data = {'color': 'black', 'font-family': 'Arial, sans serif'},
                                style_header= {'color': 'black', 'font-weight': 'bold', 'background-color': 'rgba(5, 8, 184, 0.4)', 'textAlign': 'left', 'font-family': 'Arial, sans serif'},
                                style_cell = {'textAlign': 'left', 'padding': '0px'},
                                style_table = {'height':'65vh', 'width':'95vw', 'overflow': 'auto', 'margin': 'auto', 'margin-bottom': '2vh'},
                                style_data_conditional=[
                                                            {
                                                                'if': {'row_index': 'even'},
                                                                'backgroundColor': 'rgba(5, 8, 184, 0.1)',
                                                            },
                                                            {
                                                                'if': {'row_index': 'odd'},
                                                                'backgroundColor': 'rgba(5, 8, 184, 0.4)',
                                                            },
                                                            {
                                                                'if': {'column_id': 'pub_url'},
                                                                'color': 'black',
                                                                'padding-top': '17px'
                                                            }
                                                        ])], style = {'height': '79vh', 'display': 'flex', 'flex-direction': 'column'})

@callback(
    Output('interactive-map', 'children'),
    Input('projection-selector', 'value'),
    Input('paleoenv-value', 'value'))
def create_interactive_map(projection, value_paleo):
    #df = data.read_database()
    df_ = df[df['status'] == ' ðŸŸ¢ validated']
    if value_paleo == 'All':
        df_ = df_.groupby(['latitude', 'longitude', 'location_name', 'max_age'], as_index=False).sum()[['latitude', 'longitude', 'location_name', 'number of samples', 'max_age']]
    elif value_paleo != 'All':
        df_ = df_[df_['paleoenvironment'] == value_paleo]
        df_ = df_.groupby(['latitude', 'longitude', 'location_name', 'max_age'], as_index=False).sum()[['latitude', 'longitude', 'location_name', 'number of samples', 'max_age']]
    return dcc.Graph(id = 'map', figure=plots.plot_interactive_map(df_, projection), className = 'map-style', config= {'displayModeBar': False}, responsive = True)

@callback(
    Output('hover-bar-chart', 'children'),
    Input('map', 'hoverData')
)
def create_bar_chart(hoverData):
    #df = data.read_database()
    df_ = df[df['status'] == ' ðŸŸ¢ validated']
    grouped_df = df_.groupby(['latitude', 'longitude', 'location_name', 'biosignature_cat', 'biosignature_subcat']).sum()[['number of samples' ]]
    grouped_df.reset_index(level=['biosignature_cat', 'biosignature_subcat'], inplace=True)
    if hoverData:
        loc = hoverData['points'][0]['hovertext']
        lat = hoverData['points'][0]['lat']
        lon = hoverData['points'][0]['lon']
        fig = plots.plot_bar_chart(grouped_df, lat, lon, loc)
        return dcc.Graph(figure=fig)
    else:
        return html.P('Hover on a location on the map to display more data', className = 'hover-default-text', style = {'font-family': 'Arial, sans-serif'})

@callback(
    Output('hover-pie-chart', 'children'),
    Input('map', 'hoverData')
)
def create_pie_chart(hoverData):
    #df = data.read_database()
    df_ = df[df['status'] == ' ðŸŸ¢ validated']
    grouped_df = df_.groupby(['latitude', 'longitude', 'location_name', 'detection_methods']).sum()[['number of samples' ]]
    grouped_df.reset_index(level=['detection_methods'], inplace=True)
    if hoverData:
        loc = hoverData['points'][0]['hovertext']
        lat = hoverData['points'][0]['lat']
        lon = hoverData['points'][0]['lon']
        fig = plots.plot_pie_chart(grouped_df, lat, lon, loc)
        return dcc.Graph(figure=fig)

@callback(
    Output('mars-map', 'children'),
    Input('map', 'hoverData'))
def generate_mars_map(hoverData):
    #df = data.read_database()
    df_ = df[df['status'] == ' ðŸŸ¢ validated']
    if hoverData:
        location = hoverData['points'][0]['hovertext']
        print(df_.columns)
        mars_location = df_[df_['location_name'] == location]['mars_counterpart_1'][0]
        if mars_location == 'Columbia Hills, Mars':
            return html.Img(src='/assets/mars_map_columbia.png', style = {'width': '38vw', 'height': '60vh', 'margin-left': '-4.5vw', 'margin-top': '-18vh'})
        elif mars_location == 'Eberswalde delta, Mars':
            return html.Img(src='/assets/mars_map_eberswalde.png', style = {'width': '38vw', 'height': '60vh', 'margin-left': '-4.5vw', 'margin-top': '-18vh'})
        elif mars_location == 'Meridiani Planum, Mars':
            return html.Img(src='/assets/mars_map_meridiani.png', style = {'width': '38vw', 'height': '60vh', 'margin-left': '-4.5vw', 'margin-top': '-18vh'})
    return html.Img(src='/assets/mars_map.png', style = {'width': '38vw', 'height': '60vh', 'margin-left': '-4.5vw', 'margin-top': '-18vh'})

# @callback(
#     Output("download-csv", "data"),
#     Input("btn-download-data", "n_clicks"),
#     prevent_initial_call=True,
# )
# def func(n_clicks):
#     if n_clicks > 0:
#         return dcc.send_data_frame(bio_df.to_csv, "biosignature_data.csv")

@callback(
    Output("validate-pop-up", "is_open"),
    Input("btn-validate-data", "n_clicks"),
    State("validate-pop-up", "is_open"),
    prevent_initial_call=True
)
def toggle_modal(n_clicks, is_open):
    if n_clicks > 0:
        return not is_open
    return is_open

@callback(
    Output("data-tab-buttons", "children"),
    Input("session-username", "data")
)
def generate_tab_buttons(data):
    if data['is_authorized'] == True and data['username'] == 'admin':
        #df = data.read_database()
        return  [
                        html.Button(
                             "Validate data",
                             className="doc-link-download",
                             style = {'font-family': 'Arial, sans-serif', 'font-size': '1vw', 'order': '2', 'margin-right': '1.5vw', 'margin-bottom': '-8.5vh'},
                             id = "btn-validate-data",
                             n_clicks= 0
                        ),
                        dbc.Modal(
                            [
                                dbc.ModalHeader(dbc.ModalTitle("What data do you wish to validate?"), style={'margin': 'auto'}),
                                dbc.ModalBody(children=[
                                    dash_table.DataTable(id='data-to-validate', columns = generate_data_to_validate(df)[1], data = generate_data_to_validate(df)[0], css = generate_data_to_validate(df)[2],
                                                         editable = False, style_data = {'color': 'black', 'font-family': 'Arial, sans serif'},
                                                         row_selectable = True,
                                                         selected_rows = [],
                                                         style_header= {'color': 'black', 'font-weight': 'bold', 'background-color': 'rgba(5, 8, 184, 0.4)', 'textAlign': 'left', 'font-family': 'Arial, sans serif'},
                                                         style_cell = {'textAlign': 'left', 'padding': '0px'},
                                                         style_table = {'order':'1', 'height':'20vh', 'width':'60vw', 'overflow': 'auto', 'margin': 'auto', 'margin-top': '3vh'}),
                                    html.Div(id='validate-data-modal-body', children = [], style = {'order':'2', 'display':'flex', 'flex-direction': 'column', 'align-items': 'center'}),
                                ], style = {'display':'flex', 'flex-direction': 'column', 'align-items': 'center'}),
                            ],
                            id="validate-pop-up",
                            size="xl",
                            style={'color': 'black', 'font-family': 'Arial, sans-serif', 'font-size': '1.5vw'},
                            is_open=False,
                        ),
                        html.A(
                                            "Submit data",
                                            href="/submit",
                                            className="doc-link-download",
                                            style = {'font-family': 'Arial, sans-serif', 'font-size': '1vw', 'order': '2', 'padding': '7px', 'margin-right': '12vw', 'margin-bottom': '-8.5vh', 'z-index': '1000'},
                                        )
                ]
    elif data['username'] == 'user':
        return [
            html.A(
                    "Submit data",
                    href="/submit",
                    className="doc-link-download",
                    style = {'font-family': 'Arial, sans-serif', 'font-size': '1vw', 'order': '2', 'padding': '7px', 'margin-right': '12vw', 'margin-bottom': '-8.5vh', 'z-index': '1000'},
                ),
                ]

@callback(
    Output('datatable-message', 'children'),
    Input('session-username', 'data')
)
def generate_message(data):
    if (data['is_authorized'] == True and data['username'] == 'admin') or (data['username'] == 'user'):
        return ['ðŸ‘‹ Make sure to read the ',
                html.A("contribution guidelines", href="https://alien-research.notion.site/The-Biosignature-Database-5165a9dff4e14b50ba1278c0f2012e3a", target="_blank", style = {'color': '#a0a4e4', 'font-weight': 'bold'}),
                ' before submitting any new data']
    return ['ðŸ‘‹ Log in for more features (top right corner)']

@callback(
    Output('validate-data-modal-body', 'children'),
    Input('data-to-validate', 'data'))
def generate_modal_mody(data_to_validate):
    if data_to_validate:
        return [html.Button(
                    "Validate data",
                    className="doc-link-validate",
                    style = {'font-family': 'Arial, sans-serif', 'font-size': '1vw', 'order': '1', 'margin-top': '3vh'},
                    id = "btn-validate-data-modal",
                    n_clicks= 0),
                html.P(id='validate-output', style={'order': '2', 'color': 'black', 'font-family': 'Arial, sans-serif', 'font-size': '20px', 'text-align': 'center', 'margin-top': '3vh'})]
    return "No pending data to validate"

@callback(
    Output('validate-output', 'children'),
    Input('data-to-validate', 'data'),
    Input('data-to-validate', 'selected_rows'),
    Input("btn-validate-data-modal", "n_clicks"),
    prevent_initial_call=True,
)
def func(data_to_validate, selected_rows, n_clicks):
    if n_clicks > 0:
        #df = data.read_database()
        df_to_validate = pd.DataFrame(data_to_validate)
        df_selected = df_to_validate.iloc[selected_rows]
        for index, row in df.iterrows():
            for index_, row_ in df_selected.iterrows():
                if row['biosignature_id'] == row_['biosignature_id']:
                    df.loc[index, 'status'] = ' ðŸŸ¢ validated'
        #df.to_csv('../raw_data/biosignature.csv', index=False)
        #df.to_json('data/biosignature.json', orient='records')
        return [" âœ… Your data has been successfully validated.",html.Br(),"Close the pop-up window and refresh the page to see the changes."]

@callback(
    Output('edit-pop-up-content', 'is_open'),
    Input('edit-data-btn', 'n_clicks'),
)
def generate_pop_up(n_clicks):
    if n_clicks > 0:
        return True


import numpy as np
import matplotlib.pyplot as plt
import tensorflow as tf

X_train = np.array([[1.0], [2.0]], dtype=np.float32)           #(size in 1000 square feet)
Y_train = np.array([[300.0], [500.0]], dtype=np.float32)       #(price in 1000s of dollars)

linear_layer = tf.keras.layers.Dense(units=1, activation = 'linear', )

a1 = linear_layer(X_train[0].reshape(1,1))

w, b= linear_layer.get_weights()

set_w = np.array([[200]])
set_b = np.array([100])

# set_weights takes a list of numpy arrays
linear_layer.set_weights([set_w, set_b])

a1 = linear_layer(X_train[0].reshape(1,1))
alin = np.dot(set_w,X_train[0].reshape(1,1)) + set_b

"""Calculate predictions tensorflow and numpy respectively"""
prediction_tf = linear_layer(X_train)
prediction_np = np.dot( X_train, set_w) + set_b

print(prediction_np)

"""Plot"""
x = np.linspace(0, 2, 100)
plt.plot(X_train, prediction_tf)
plt.plot(X_train, prediction_np)
plt.scatter(X_train, Y_train)
plt.show()

"""
fig, ax = plt.subplots(1,1)
ax.scatter(X_train, Y_train, marker='x', c='r', label="Data Points")
ax.legend( fontsize='xx-large')
ax.set_ylabel('Price (in 1000s of dollars)', fontsize='xx-large')
ax.set_xlabel('Size (1000 sqft)', fontsize='xx-large')
plt.show()
"""


import sys
import json
import datetime
import logging
import os
import uuid
# pylint: disable=import-error
from helpers.response import ResponseHelper
from helpers.utils import convert_datetime_to_epoch
from helpers.dynamodb_helper import DynamoDbHelper

logger = logging.getLogger()
logger.setLevel(logging.INFO)

def lambda_handler(event, context):
    logger.info(event)
    status_code = 200
    message = "Record added successfully"
    record_id = None

    try:
        user_context = event["requestContext"]["authorizer"]
        question_id = event["pathParameters"]["question_id"]
        community_id = event["pathParameters"]["community_id"]
        body = json.loads(event['body'])

        if DynamoDbHelper.is_exists(os.environ["QUESTION_TABLE_NAME"], question_id):
            question_record = \
                DynamoDbHelper.get(os.environ['QUESTION_TABLE_NAME'],
                                   {"id": question_id, "community_id": community_id})[
                    "number_of_comments"]
            comments_number = question_record.get("number_of_comments", default=0)

            record_id = add_record(body, question_id, user_context, comments_number, community_id)
        else:
            status_code = 404
            message = "Given question doesn't exists"
    except Exception as exception:
        logger.error("Exception: {}".format(exception), exc_info=sys.exc_info())
        status_code = 500
        message = "Process has encountered an internal server exception."

    response = {
        "message": message
    }

    if not record_id is None:
        response["id"] = record_id

    return ResponseHelper.format_response(status_code, response)

def add_record(body, question_id, user_context, comments_number: int, community_id: str):
    current_datetime = datetime.datetime.now()
    body["id"] = str(uuid.uuid4())
    body["created_time"] = str(current_datetime)
    body["created_time_epoch"] = int(convert_datetime_to_epoch(current_datetime))
    body["question_id"] = question_id
    body["user_id"] = user_context["id"]
    body["user_email"] = user_context["email"]

    DynamoDbHelper.insert(os.environ["ANSWER_TABLE_NAME"], body)
    DynamoDbHelper.update_field(table_name=os.environ["QUESTION_TABLE_NAME"], key_name='id',
                                key_value=question_id,
                                field_name='number_of_comments', field_value=comments_number + 1,
                                community_id=community_id)

    return body["id"]


import pook
import pytest
import os

from shutil import rmtree
from data_node import app as data_node, FILE_STORE

@pytest.fixture
def client():
    data_node.config["TESTING"] = True

    with data_node.test_client() as client:
        yield client

def init_storage_test():
    if os.path.exists(FILE_STORE):
        rmtree(FILE_STORE)
        os.mkdir(FILE_STORE)
    else:
        os.mkdir(FILE_STORE)

def test_filesystem_delete(client):
    init_storage_test()

    os.mkdir(os.path.join(FILE_STORE, "test"))
    assert len(os.listdir(FILE_STORE)) == 1

    client.delete("filesystem")
    assert os.path.exists(FILE_STORE)
    assert len(os.listdir(FILE_STORE)) == 0

def test_file_get(client):
    init_storage_test()

    fpath = os.path.join(FILE_STORE, "1")
    f = open(fpath, "w")
    data = "AAAAAAAAAABBBBBBBBBBBCCCCCCCCCCCCCCCCCCC"
    f.write(data)
    f.close()

    resp = client.get("/file?filename=1")
    assert resp.status_code == 200
    assert resp.data.decode("utf-8") == data

    resp = client.get("/file?filename=2")
    assert resp.status_code == 404

    os.remove(fpath)

@pook.on
def test_file_put(client):
    init_storage_test()
    mock = pook.post(
        "http://localhost:3030/file_created?file_id=1&port=2020", reply=200
    )

    assert not os.path.exists(os.path.join(FILE_STORE, "1"))
    data = "AAAAAAAAAAABBBBBBBBBBCCCCCCCCCCCCCC"

    resp = client.post("/file?filename=1", data=data)
    assert resp.status_code == 201

    resp = client.post("/file?filename=../b.txt", data=data)
    assert resp.status_code == 400

    resp = client.post("/file?filename=1", data=data)
    assert resp.status_code == 400

def test_file_delete(client):
    init_storage_test()
    fpath = os.path.join(FILE_STORE, "1")
    open(fpath, "w")

    assert client.get("/file?filename=1").status_code == 200

    assert client.delete("/file?filename=1").status_code == 200
    assert not os.path.exists(fpath)


import cv2
import numpy as np

#Йщох╣■¤Б╦сие
def aHash(img):
    img = cv2.resize(img, (8,8), interpolation=cv2.INTER_CUBIC)
    gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
    s = 0
    hash_str = ''
    for i in range(8):
        for j in range(8):
            s = s + gray[i,j]
    avg = s/64
    for i in range(8):
        for j in range(8):
            if gray[i,j] > avg:
                hash_str = hash_str + '1'
            else:
                hash_str = hash_str + '0'
    return hash_str

#▓Ьох╣■¤Б╦сие
def dHash(img):
    img = cv2.resize(img, (9,8), interpolation=cv2.INTER_CUBIC)
    gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
    hash_str = ''
    for i in range(8):
        for j in range(8):
            if gray[i,j] > gray[i,j+1]:
                hash_str = hash_str+'1'
            else:
                hash_str = hash_str+'0'
    return hash_str

def cmpHash(hash1, hash2):
    n = 0
    if len(hash1) != len(hash2):
        return -1
    for i in range(len(hash1)):
        if hash1[i] != hash2[i]:
            n = n+1
    return n

img1 = cv2.imread('lenna.png')
img2 = cv2.imread('lenna_PepperandSalt.png')
hash1 = aHash(img1)
hash2 = aHash(img2)
print(hash1)
print(hash2)
n = cmpHash(hash1,hash2)
print("Йщох╣■¤Б╦сие¤Я╦кХ╚Б║", n)

hash2 = dHash(img1)
hash2 = dHash(img2)
print(hash1)
print(hash2)
n = cmpHash(hash1,hash2)
print("▓Ьох╣■¤Б╦сие¤Я╦кХ╚Б║", n)


#!/usr/bin/env python3
# -*-coding:utf-8 -*-
# =============================================================================
"""
@Author        :   Yujie He
@File          :   eval_crowd.py
@Date created  :   2021/11/02
@Maintainer    :   Yujie He
@Email         :   yujie.he@epfl.ch
"""
# =============================================================================
"""
The module provides the evaluation pipeline includeing the computation of
crowd-related metrics (min_dist, crowd density within 5/10m, nomalized
proximity and corresponding visualization.
The emulation results is exported with suffix as "_crowd_eval.npy".
"""
# =============================================================================
"""
TODO:
1. compare with detected pedestrain from the rosbag!
2. use try/except when loading files
"""
# =============================================================================

import os
import argparse
import math
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt

from qolo.core.crowdbot_data import (
    CrowdBotDatabase,
    CrowdbotExpParam,
    CROWDBOT_EVAL_TOOLKIT_DIR,
)
from qolo.utils.notebook_util import values2color_list
from qolo.utils.res_plot_util import save_cd_img, save_md_img, save_cd_img_single
from qolo.metrics.metric_crowd import compute_crowd_metrics, compute_norm_prox

# cross-zero checking: https://stackoverflow.com/a/29674950/7961693
def zero_crossing_check(data):
    crossing_idx = np.where(np.diff(np.signbit(data)))[0]
    less_than_zero = math.ceil(float(crossing_idx.shape[0]) / 2)
    return less_than_zero

#%% main function
if __name__ == "__main__":

    data_params_path = os.path.join(
        CROWDBOT_EVAL_TOOLKIT_DIR, "data", "data_params.yaml"
    )

    parser = argparse.ArgumentParser(description="Evaluate crowd characteristics")

    parser.add_argument(
        "-f",
        "--folder",
        default="0424_shared_control",
        type=str,
        help="different subfolder in rosbag/ dir",
    )
    parser.add_argument(
        "--params_path",
        default=data_params_path,
        type=str,
        help="path to dataset parameters",
    )
    parser.add_argument(
        "--save_img",
        dest="save_img",
        action="store_true",
        help="plot and save crowd density image",
    )
    parser.set_defaults(save_img=True)
    parser.add_argument(
        "--overwrite",
        dest="overwrite",
        action="store_true",
        help="Whether to overwrite existing rosbags (default: false)",
    )
    parser.set_defaults(overwrite=False)
    parser.add_argument(
        "--replot",
        dest="replot",
        action="store_true",
        help="Whether to re-plot existing images (default: false)",
    )
    parser.set_defaults(replot=False)
    parser.add_argument(
        "--goal_dist", default=30.0, type=float, help="The length to travel in the test"
    )
    args = parser.parse_args()

    cb_data = CrowdBotDatabase(args.folder)

    print("Starting evaluating crowd from {} sequences!".format(cb_data.nr_seqs()))

    all_data_params = CrowdbotExpParam(args.params_path)
    date = args.folder[:4]
    control_type = args.folder[5:]
    data_params = all_data_params.get_params(date, control_type)
    # {'goal_dist': float, 'vel_user_max': float, 'omega_user_max': float}
    print("# Experiment data:", date)
    print("# Experiment control type:", control_type)
    print("# Experiment settings:", data_params)

    eval_res_dir = os.path.join(cb_data.metrics_dir)

    if not os.path.exists(eval_res_dir):
        print("Result images and npy will be saved in {}".format(eval_res_dir))
        os.makedirs(eval_res_dir, exist_ok=True)

    for seq_idx in range(cb_data.nr_seqs()):

        seq = cb_data.seqs[seq_idx]
        print(
            "({}/{}): {} with {} frames".format(
                seq_idx + 1, cb_data.nr_seqs(), seq, cb_data.nr_frames(seq_idx)
            )
        )

        # load twist, pose2d
        twist_dir = os.path.join(cb_data.source_data_dir, "twist")
        qolo_twist_path = os.path.join(twist_dir, seq + "_twist_raw.npy")  # _twist
        if not os.path.exists(qolo_twist_path):
            print("ERROR: Please extract twist_stamped by using twist2npy.py")
        qolo_twist = np.load(qolo_twist_path, allow_pickle=True).item()

        pose2d_dir = os.path.join(cb_data.source_data_dir, "pose2d")
        qolo_pose2d_path = os.path.join(pose2d_dir, seq + "_pose2d.npy")
        if not os.path.exists(qolo_pose2d_path):
            print("ERROR: Please extract pose2d by using pose2d2npy.py")
        qolo_pose2d = np.load(qolo_pose2d_path, allow_pickle=True).item()

        # load _path_eval.npy
        path_eval_filepath = os.path.join(eval_res_dir, seq, seq + "_path_eval.npy")
        if not os.path.exists(path_eval_filepath):
            print("ERROR: Please extract twist_stamped by using eval_qolo_path.py")
        path_eval_dict = np.load(path_eval_filepath, allow_pickle=True).item()

        # dest: seq+'_crowd_eval.npy' file in eval_res_dir
        crowd_eval_npy = os.path.join(eval_res_dir, seq, seq + "_crowd_eval.npy")

        # only for plotting function update!
        if args.replot:
            crowd_eval_dict = np.load(crowd_eval_npy, allow_pickle=True).item()

            # recalculate virtual_collision
            min_dist_list = np.array(crowd_eval_dict["min_dist"])
            start_idx_ = path_eval_dict['start_idx']
            end_idx_ = path_eval_dict['end_idx']
            min_dist_list = min_dist_list[start_idx_ : end_idx_ + 1]
            # only consider the point that more than zero
            min_dist_list_ = min_dist_list[min_dist_list > 0]
            avg_min_dist = np.sum(min_dist_list_) / min_dist_list_.shape[0]

            # consider the number of crossing zero from positive to negative
            virtual_collision = zero_crossing_check(min_dist_list)

            crowd_eval_dict['avg_min_dist'] = avg_min_dist
            crowd_eval_dict['virtual_collision'] = virtual_collision
            print("virtual_collision:", crowd_eval_dict['virtual_collision'])
            np.save(crowd_eval_npy, crowd_eval_dict)

            # figure1: crowd density
            _, color_unique = values2color_list(
                [0, 1, 2],
                cmap_name='hot',
                given_values=[0.2, 0.55, 0.66],
                reverse=False,
            )

            save_cd_img(
                crowd_eval_dict,
                path_eval_dict,
                eval_res_dir,
                seq,
                fmt='pdf',
                add_title=False,
                use_serif=True,
                color_list=color_unique,
                color_vertical='navy',
            )
            # with single plots
            # save_cd_img_single(
            #     crowd_eval_dict,
            #     path_eval_dict,
            #     eval_res_dir,
            #     seq,
            #     dist=2.5,
            #     fmt='pdf',
            #     linecolor="navy",
            #     use_serif=True,
            # )

            # figure2: min. dist.
            save_md_img(crowd_eval_dict, path_eval_dict, eval_res_dir, seq)

            print("Replot images!")
        else:
            if (not os.path.exists(crowd_eval_npy)) or (args.overwrite):

                # timestamp can be read from lidars/ folder
                lidar_stamp_dir = os.path.join(cb_data.source_data_dir, "timestamp")
                stamp_file_path = os.path.join(lidar_stamp_dir, seq + "_stamped.npy")
                lidar_stamped = np.load(
                    stamp_file_path,
                    allow_pickle=True,
                ).item()
                ts = lidar_stamped.get("timestamp")

                # targeted metrics and correspoding dtype
                attrs = (
                    "all_det",
                    "within_det2_5",
                    "within_det3",
                    "within_det5",
                    "within_det10",
                    "crowd_density2_5",
                    "crowd_density3",
                    "crowd_density5",
                    "crowd_density10",
                    "min_dist",
                )
                dtypes = (
                    np.uint8,
                    np.uint8,
                    np.uint8,
                    np.uint8,
                    np.uint8,
                    np.float32,
                    np.float32,
                    np.float32,
                    np.float32,
                    np.float32,
                )

                crowd_eval_list_dict = {k: [] for k in attrs}

                num_msgs_between_logs = 100
                nr_frames = cb_data.nr_frames(seq_idx)

                for fr_idx in range(nr_frames):

                    _, _, _, trks = cb_data[seq_idx, fr_idx]

                    metrics = compute_crowd_metrics(
                        trks, virtual_radius=data_params["virtual_radius"]
                    )

                    if fr_idx % num_msgs_between_logs == 0 or fr_idx >= nr_frames - 1:
                        print(
                            "Seq {}/{} - Frame {}/{}: Crowd density within 2.5/3/5/10m: {:.2f} / {:.2f} / {:.2f} / {:.2f}".format(
                                seq_idx + 1,
                                cb_data.nr_seqs(),
                                fr_idx + 1,
                                nr_frames,
                                metrics[5],
                                metrics[6],
                                metrics[7],
                                metrics[8],
                            )
                        )

                    # update value for each attr
                    for idx, val in enumerate(metrics):
                        crowd_eval_list_dict[attrs[idx]].append(val)

                # evaluate the metrics through the sequence
                crowd_eval_dict = {
                    name: np.asarray(crowd_eval_list_dict[attrs[idx]], dtype=dtype)
                    for idx, (name, dtype) in enumerate(zip(attrs, dtypes))
                }

                # normalized proximity
                crowd_eval_dict.update(
                    {"normalized_proximity": compute_norm_prox(metrics[5])}
                )

                # calculate the avg for each matrix except avg_min_dist
                for attr in attrs:
                    if attr == "min_dist":
                        continue
                    avg_attr = "avg_" + attr
                    crowd_eval_dict.update(
                        {avg_attr: np.average(crowd_eval_dict[attr])}
                    )

                # calculate avg_min_dist
                min_dist_list = np.array(crowd_eval_dict["min_dist"])
                start_idx_ = path_eval_dict['start_idx']
                end_idx_ = path_eval_dict['end_idx']
                min_dist_list = min_dist_list[start_idx_ : end_idx_ + 1]
                # only consider the point that more than zero
                min_dist_list_ = min_dist_list[min_dist_list > 0]
                avg_min_dist = np.sum(min_dist_list_) / min_dist_list_.shape[0]

                # consider the number of crossing zero from positive to negative
                virtual_collision = zero_crossing_check(min_dist_list)

                crowd_eval_dict.update({'avg_min_dist': avg_min_dist})
                crowd_eval_dict.update({'virtual_collision': virtual_collision})
                print("virtual_collision:", crowd_eval_dict['virtual_collision'])

                # max and std of crowd density within 2.5m/5m
                crowd_eval_dict.update(
                    {
                        'max_crowd_density2_5': np.max(
                            crowd_eval_dict['crowd_density2_5']
                        )
                    }
                )
                crowd_eval_dict.update(
                    {
                        'std_crowd_density2_5': np.std(
                            crowd_eval_dict['crowd_density2_5']
                        )
                    }
                )

                crowd_eval_dict.update(
                    {'max_crowd_density5': np.max(crowd_eval_dict['crowd_density5'])}
                )
                crowd_eval_dict.update(
                    {'std_crowd_density5': np.std(crowd_eval_dict['crowd_density5'])}
                )

                # other attributes
                crowd_eval_dict.update({"timestamp": ts})

                np.save(crowd_eval_npy, crowd_eval_dict)

                if args.save_img:
                    # figure1: crowd density
                    save_cd_img(crowd_eval_dict, path_eval_dict, eval_res_dir, seq)

                    # figure2: min. dist.
                    save_md_img(crowd_eval_dict, path_eval_dict, eval_res_dir, seq)

            else:
                print(
                    "Detecting the generated {} already existed!".format(crowd_eval_npy)
                )
                print(
                    "Will not overwrite. If you want to overwrite, use flag --overwrite"
                )
                continue

    print("Finish crowd evaluation!")