import shutil
import tempfile

from http import HTTPStatus

from django.conf import settings
from django.core.files.uploadedfile import SimpleUploadedFile
from django.contrib.auth import get_user_model
from django.test import Client, TestCase
from django.urls import reverse
from posts.models import Group, Post

# python manage.py test posts.tests.test_forms -v 0

User = get_user_model()

class FormTest(TestCase):
    """Проверка формы создания нового поста"""
    @classmethod
    def setUpClass(cls):
        super().setUpClass()
        # Создаем временную папку для медиа-файлов;
        # на момент теста медиа папка будет переопределена
        settings.MEDIA_ROOT = tempfile.mkdtemp(dir=settings.BASE_DIR)

    @classmethod
    def tearDownClass(cls):
        # Модуль shutil - библиотека Python с инструментами
        # для управления файлами и директориями:
        # создание, удаление, копирование, перемещение, изменение папок|файлов
        # Метод shutil.rmtree удаляет директорию и всё её содержимое
        shutil.rmtree(settings.MEDIA_ROOT, ignore_errors=True)
        super().tearDownClass()

    def setUp(self):
        self.user = User.objects.create_user(username='Pascha')
        self.authorized_client = Client()
        self.authorized_client.force_login(self.user)
        self.group = Group.objects.create(
            title='Тестовая группа',
            slug='test-slug',
            description='тестовое описание')
        self.another_group = Group.objects.create(
            title='Другая тестовая группа',
            slug='another_test-slug',
            description='тестовое описание')
        self.post = Post.objects.create(
            text='пробный текст',
            author=self.user,
            group=self.group)
        self.guest_client = Client()

    def test_create_post(self):
        """Валидная форма создает пост и перенаправляет на главную страницу"""
        posts_count = Post.objects.count()
        # Для тестирования загрузки изображений
        # берём байт-последовательность картинки,
        # состоящей из двух пикселей: белого и чёрного
        small_gif = (
            b'\x47\x49\x46\x38\x39\x61\x02\x00'
            b'\x01\x00\x80\x00\x00\x00\x00\x00'
            b'\xFF\xFF\xFF\x21\xF9\x04\x00\x00'
            b'\x00\x00\x00\x2C\x00\x00\x00\x00'
            b'\x02\x00\x01\x00\x00\x02\x02\x0C'
            b'\x0A\x00\x3B'
        )
        uploaded = SimpleUploadedFile(
            name='small.gif',
            content=small_gif,
            content_type='image/gif'
        )
        form_data = {
            'text': 'Тестовый текст',
            'group': self.group.id,
            'image': uploaded,
        }
        response = self.authorized_client.post(
            reverse('new_post'),
            data=form_data,
            follow=True
        )
        self.assertRedirects(response, reverse('index'))
        self.assertEqual(Post.objects.count(), posts_count + 1)
        self.assertEqual(response.status_code, HTTPStatus.OK)
        # проверям по полям, что пост создался.
        # [0] так как в мета модели ordering = ['-pub_date']
        post = Post.objects.all()[0]
        self.assertEqual(post.text, form_data['text'])
        self.assertEqual(post.author, self.user)
        self.assertEqual(post.group, self.group)
        self.assertEqual(post.image, 'posts/small.gif')
        # Можно еще проверить наличие созданного поста:
        #  т.о assertTrue(Post.objects.filter(.., group=self.group).exists())

    def test_edit_post(self):
        """
        При редактировании поста через форму
        изменяется соответствующая запись в базе данных
        """
        form_data = {
            'text': 'Пробный текст изменен', 'group': self.another_group.id, }
        response = self.authorized_client.post(
            reverse('edit', kwargs={'username': 'Pascha', 'post_id': '1'}),
            data=form_data,
            follow=True
        )
        self.assertRedirects(response, reverse(
            'post', kwargs={'username': 'Pascha', 'post_id': '1'}))
        self.assertTrue(
            Post.objects.filter(
                text='Пробный текст изменен',
                author=self.user,
                group=self.another_group).exists())
        post = self.post
        response1 = self.authorized_client.get(reverse(
            'group', kwargs={'slug': 'test-slug'}))
        response2 = self.authorized_client.get(reverse(
            'group', kwargs={'slug': 'another_test-slug'}))
        post_count1 = response1.context['page'].count(post)
        post_count2 = response2.context['page'].count(post)
        posts_count = [
            (post_count1, 0, 'Тестовая группа'),
            (post_count2, 1, 'Другая тестовая группа')]
        for posts, count, group in posts_count:
            with self.subTest(group=group):
                self.assertEqual(posts, count)

    def test_guest_client_cant_create_post(self):
        """Неавторизованный пользователь не может публиковать пост"""
        posts_count = Post.objects.count()
        form_data = {'text': 'Не авторизованный текст'}
        response = self.guest_client.post(
            reverse('new_post'), data=form_data, follow=True)
        self.assertRedirects(response, '/auth/login/?next=/new/')
        # проверям, что количество постов не изменилось
        self.assertEqual(Post.objects.count(), posts_count)


import argparse
import json
import os
import sys
from typing import Dict, Any

import WDL

_output_mapping = [
    (WDL.Type.File(),
     "file"),
    (WDL.Type.Array(
        WDL.Type.File(), nonempty=True),
     "files"),
    (WDL.Type.Pair(
        WDL.Type.File(),
        WDL.Type.Map(
            (WDL.Type.String(), WDL.Type.String()))),
     "file-with-labels"),
    (WDL.Type.Pair(
        WDL.Type.Array(
            WDL.Type.File(), nonempty=True),
        WDL.Type.Map(
            (WDL.Type.String(),
             WDL.Type.String()))),
     "files-with-labels"),
    (WDL.Type.Array(
        WDL.Type.Pair(
            WDL.Type.File(),
            WDL.Type.Map(
                (WDL.Type.String(),
                 WDL.Type.String()))), nonempty=True),
     "files-with-labels"),
    (WDL.Type.Boolean(),
     "quality-control"),
    (WDL.Type.File(
        optional=True),
     "optional-file"),
    (WDL.Type.Array(WDL.Type.File(), optional=True, nonempty=True),
     "optional-files"),
    (WDL.Type.Array(WDL.Type.File(), optional=True, nonempty=False),
     "optional-files"),
    (WDL.Type.Array(
        WDL.Type.File(),
        nonempty=False),
     "optional-files"),
    (WDL.Type.Pair(
        WDL.Type.File(),
        WDL.Type.Map(
            (WDL.Type.String(),
             WDL.Type.String())),
        optional=True),
     "optional-file-with-labels"),
    (WDL.Type.Pair(
        WDL.Type.Array(WDL.Type.File(), nonempty=True),
        WDL.Type.Map(
            (WDL.Type.String(),
             WDL.Type.String())),
        optional=True),
     "optional-files-with-labels"),
    (WDL.Type.Boolean(
        optional=True),
     "optional-quality-control"),
]

def _map_input(wdl_type: WDL.Type.Base, structures: Dict[str, Any]):
    if wdl_type.optional:
        return {
            "is": "optional",
            "inner": _map_inner_input(
                wdl_type,
                structures)}
    else:
        return _map_inner_input(wdl_type, structures)

def _map_inner_input(wdl_type: WDL.Type.Base, structures: Dict[str, Any]):
    if isinstance(wdl_type, WDL.Type.Array):
        (inner,) = wdl_type.parameters
        return {"is": "list", "inner": _map_input(inner, structures)}
    elif isinstance(wdl_type, WDL.Type.Pair):
        (left_type, right_type) = wdl_type.parameters
        return {
            "is": "pair", "left": _map_input(
                left_type, structures), "right": _map_input(
                right_type, structures)}
    elif isinstance(wdl_type, WDL.Type.StructInstance):
        return {"is": "object", "fields": structures[wdl_type.type_name]}
    elif isinstance(wdl_type, WDL.Type.Map):
        (key_type, value_type) = wdl_type.parameters
        return {
            "is": "dictionary", "key": _map_input(
                key_type, structures), "value": _map_input(
                value_type, structures)}
    elif isinstance(wdl_type, WDL.Type.Boolean):
        return "boolean"
    elif isinstance(wdl_type, WDL.Type.Directory):
        return "directory"
    elif isinstance(wdl_type, WDL.Type.File):
        return "file"
    elif isinstance(wdl_type, WDL.Type.Float):
        return "floating"
    elif isinstance(wdl_type, WDL.Type.Int):
        return "integer"
    elif isinstance(wdl_type, WDL.Type.String):
        return "string"
    raise ValueError(f"No conversion for {wdl_type}")

def _map_output(wdl_type: WDL.Type.Base, allow_complex: bool,
                structures: WDL.Env.Bindings[WDL.StructTypeDef]):
    for (vidarr_wdl_type, vidarr_type) in _output_mapping:
        if wdl_type == vidarr_wdl_type:
            return vidarr_type
    if allow_complex and isinstance(wdl_type, WDL.Type.Array):
        (inner,) = wdl_type.parameters
        if isinstance(inner, WDL.Type.StructInstance):
            keys = {}
            outputs = {}
            for (member_name,
                 member_type) in structures[inner.type_name].members.items():
                if member_type == WDL.Type.Int():
                    keys[member_name] = "INTEGER"
                elif member_type == WDL.Type.String():
                    keys[member_name] = "STRING"
                else:
                    outputs[member_name] = _map_output(
                        member_type, False, structures)
            return {"is": "list", "keys": keys, "outputs": outputs}
    raise ValueError(
        f"Vidarr cannot process output type {wdl_type} in output.")

def parse(wdl_file_path: str) -> Dict[str, Any]:
    """
    Read a WDL file and convert it into a Vidarr workflow definition

    :param wdl_file_path: the path to the WDL file; this must be a path on disk in case the WDL file imports other files
    :return: the Vidarr configuration object
    """
    return convert(WDL.load(wdl_file_path))

def convert(doc: WDL.Document) -> Dict[str, Any]:
    """
    Convert a parsed WDL file into a Vidarr workflow definition
    :param doc:  the parsed WDL file
    :return: the Vidarr configuration object
    """
    workflow_name = doc.workflow.name
    structures = {}
    unprocessed_structs = [s for s in doc.struct_typedefs]
    while unprocessed_structs:
        structs_needing_unprocessed = []
        for structure in unprocessed_structs:
            try:
                structures[structure.name] = {member_name: _map_input(member_type, structures) for (
                    member_name, member_type) in structure.value.members.items()}
            except KeyError:
                structs_needing_unprocessed.append(structure)
        assert len(structs_needing_unprocessed) < len(unprocessed_structs)
        unprocessed_structs = structs_needing_unprocessed

    workflow_inputs = {}

    def read_output(output: WDL.Decl):
        output_metadata = doc.workflow.meta.get(
            "output_meta", {}).get(
            output.name, {})
        if isinstance(
                output_metadata,
                dict) and "vidarr_type" in output_metadata:
            return output_metadata["vidarr_type"]
        else:
            return _map_output(
                output.type, True, doc.struct_typedefs)

    for wf_input in (doc.workflow.available_inputs or []):
        meta = doc.workflow.parameter_meta.get(wf_input.name)

        if meta and isinstance(meta, dict) and "vidarr_type" in meta:
            vidarr_type = meta["vidarr_type"]
        else:
            vidarr_type = _map_input(wf_input.value.type, structures)

        # Optional should be processed after retry because it should be okay to omit a retriable value and use a
        # singular default.
        if meta and isinstance(meta, dict) and meta.get("vidarr_retry", False):
            vidarr_type = {"is": "retry", "inner": vidarr_type}

        if wf_input.value.expr:
            vidarr_type = {"is": "optional", "inner": vidarr_type}

        workflow_inputs[doc.workflow.name + "." + wf_input.name] = vidarr_type

    workflow = {
        'language': 'WDL_' + str(
            doc.wdl_version).replace(
            ".",
            "_"),
        'outputs': {
            workflow_name + "." + output.name:
                read_output(output)
            for output in doc.workflow.outputs},
        'parameters': workflow_inputs,
        'workflow': doc.source_text,
        'accessoryFiles': {
            imported.uri: imported.doc.source_text for imported in doc.imports}}
    return workflow

def main():
    parser = argparse.ArgumentParser(
        formatter_class=argparse.ArgumentDefaultsHelpFormatter)
    parser.add_argument(
        "-i",
        "--input-wdl-path",
        required=True,
        help="Source wdl path")
    parser.add_argument(
        "-o",
        "--output-path",
        required=False,
        help="Output a file with contents of the workflow parameter dict")
    args = parser.parse_args()

    if args.output_path:
        workflow = parse(args.input_wdl_path)
        parent_dir = os.path.dirname(args.output_path)
        if parent_dir:
            os.makedirs(parent_dir, exist_ok=True)
        with open(args.output_path, "w") as output_file:
            json.dump(workflow, output_file)
    else:
        json.dump(
            parse(
                args.input_wdl_path),
            sys.stdout,
            indent=4,
            sort_keys=True)

if __name__ == "__main__":
    main()


"""
Here is the link to the data. You really only need the file
jgoethe.xml, the entities and the schema, but I was to lazy to sort it
out. The file contains the works of young Goethe (basically all he
wrote up to his 26th birthday) and our commentary and additional
texts, for example a mythological dictionary and the Luther bibel. All
references to the bibel are internal links but it doesn't always say
explicitly bibel in the notes text. So if you want to extract *all*
references, the safest way is probably to collect all ref target ids
in the bibel and then search for them in the rest of the corpus.
"""

import re
import string
import collections
import os
from lxml import etree
import tqdm

ORIGINAL_PATH = './data/jgoethe.xml'
PATH = './data/jgoethe.utf8.noents.xml'
PARSER = etree.XMLParser()

def bs4_transform_entities(path=ORIGINAL_PATH, output=PATH):
    import bs4
    with open(path, encoding='iso-8859-1') as f:
        # ignore first 10 lines (entities and stuff)
        for _ in range(10):
            next(f)
        t = bs4.BeautifulSoup(f.read(), 'lxml').find('tei.2')
    with open(output, 'w') as f:
        f.write(str(t))

if not os.path.isfile(PATH):
    bs4_transform_entities()

def get_bible_verses(tree):
    # # check all in bible
    # bible = tree.xpath('//div1[@id="JG46062"]')[0]
    # for verse in tqdm.tqdm(verses, total=len(verses)):
    #     assert bible.xpath('//*[@id="{}"]'.format(verse['verse_id'])), verse['verse_id']
    verses = []
    for verse in tree.xpath('//l[@rend="Bibelvers"]'):
        text = verse.find('anchor').tail.strip()
        verses.append({
            "anchor": verse.find('anchor').attrib['id'],
            "verse_id": verse.attrib['id'],
            "verse_num": int(verse.attrib['n']),
            "text": text
        })

    return verses

def remove_bible_verses(tree):
    bible = tree.xpath('//div1[@id="JG46062"]')[0]
    bible.getparent().remove(bible)

# look for references in the tree
def read_refs(path='refs.txt'):
    with open(path) as f:
        refs = list(set([line.strip() for line in f]))
    return refs

def extract_ref_text(text, note):
    # extract actual text
    if not note.strip().endswith(']'):
        return

    # normalize whitespace
    text, note = ' '.join(text.strip().split()), ' '.join(note.strip().split())
    note = note[:-1]
    if note.startswith('['):
        note = note[1:]
    if '[...]' in note:
        start, *_, end = note.split('[...]')
        start, end = start.strip(), end.strip()
        if not end:
            return start

        if start not in text or end not in text:
            print(repr(text))
            return text

        start_index = text.index(start)
        end_index = text.index(end)

        return text[start_index: end_index + len(end)]

    else:
        return note

def get_ref_text(ref):
    p = ref.getparent()

    if p.getparent() is None:
        return
    if p.find('hi') is None:
        return

    if p.getprevious() is not None:
        text = p.getprevious().tail
    else:
        text = ''
        if p.getparent().getprevious() is not None:
            text += p.getparent().getprevious().text or ''
        text += p.getparent().text
        if p.getparent().getnext() is not None:
            text += p.getparent().getnext().text or ''

    note = p.find('hi').text

    return extract_ref_text(text, note)

def get_dataset(tree):
    targets = set(read_refs())
    refs, errors, nonote = [], [], []
    by_anchor = collections.defaultdict(list)

    # get verses
    verses = get_bible_verses(tree)

    # remove verses to avoid getting bible verses as matches
    remove_bible_verses(tree)

    for verse in tqdm.tqdm(verses):
        if verse['anchor'] not in targets:
            continue
        for ref in tree.xpath('//ref[@target="{}"]'.format(verse['anchor'])):
            if ref.getparent().tag != 'note':
                nonote.append((verse, ref))
                continue

            ref_text = get_ref_text(ref)
            if not ref_text:
                errors.append((verse, ref))
                continue

            if verse['anchor'] in by_anchor and ref_text in by_anchor[verse['anchor']]:
                print("Duplicate")
                continue

            refs.append(
                {'anchor': verse['anchor'],
                 'target': verse,
                 'ref': ref,
                 'source': ref_text})

            by_anchor[verse['anchor']].append(refs[-1]['source'])

    return refs, errors, nonote

regex = re.compile('(^[{chars}]|[{chars}]+$)'.format(
    chars=''.join(re.escape(c) for c in string.punctuation)))

def strip_punct(s):
    return [re.sub(regex, '', w) for w in s]

if __name__ == '__main__':
    tree = etree.parse(PATH, PARSER)
    bible = get_bible_verses(tree)
    refs, errors, nonote = get_dataset(tree)

    # remove single word references: "wie Kain"
    for r in refs:
        if len(r['source'].split()) < 3:
            print(r)

    refs = [r for r in refs if len(r['source'].split()) > 2]

    with open('goethe-gold.csv', 'w') as f:
        for ref in refs:
            sid1, sid2 = '_', ref['anchor'] + '-' + ref['target']['verse_id']
            s1 = ref['source']
            s2 = ref['target']['text']
            s1 = ' '.join(strip_punct(s1.split()))
            s2 = ' '.join(strip_punct(s2.split()))
            l1, l2 = '_', '_'
            f.write('\t'.join([sid1, sid2, s1, s2, l1, l2]) + '\n')

    anchors = set([ref['anchor'] for ref in refs])
    with open('goethe-background.csv', 'w') as f:
        for verse in bible:
            if verse['anchor'] in anchors:
                print("Duplicate")
                continue
            text = ' '.join(strip_punct(verse['text'].split()))

            sid = verse['anchor'] + '-' + verse['verse_id']
            f.write('\t'.join([sid, text, '_']) + '\n')

# for ref in refs:
#     print("ANCHOR:", ref["anchor"])
#     print("target ->", ref["target"]["text"])
#     print("source ->", ref['source'])
#     print()
#     print()

# mrefs = [r for r in refs if r['anchor'] == 'AMOSBHAB']

# for r in refs:
#     if not r['target_note'].strip().endswith(']'):
#         print(r['target_note'])


from bootstrap_datepicker_plus import DatePickerInput
from django import forms
from django.contrib.auth.models import User
from django.core import validators
from django.shortcuts import get_object_or_404
from django.utils.translation import gettext_lazy as _
from main_app.models import Region
from places.models import Place
from photo_reports.models import PhotoReport, Photo
from video_reports.models import VideoReport

class PhotoReportForm(forms.ModelForm):
    """ PhotoReport Form """
    class Meta:
        model = PhotoReport
        fields = [
            'author',
            'authors',
            'place',
            'date',
            'title',
            'pay_to_photographer',
            'pay_to_portal',
            'is_plus_18',
            'is_posted',
        ]

    def __init__(self, request, *args, **kwargs):
        super(PhotoReportForm, self).__init__(*args, **kwargs)
        region = get_object_or_404(Region, slug=request.session['user_region'])
        authors = User.objects.filter(
            user_access__region=region, user_access__is_photographer=True)
        places = Place.objects.filter(region=region, is_closed=False)

        AUTHOR_CHOICES = []
        for author in authors:
            if author.get_full_name():
                AUTHOR_CHOICES.append((str(author.id), author.get_full_name()))
            else:
                AUTHOR_CHOICES.append((str(author.id), author.get_username()))

        self.fields['author'].choices = AUTHOR_CHOICES
        self.fields['author'].label = _('Primary Photographer')

        self.fields['authors'].choices = AUTHOR_CHOICES
        self.fields['authors'].label = _('More Photographers')
        self.fields['authors'].help_text = _('Hold down "Control", or "Command" on a Mac, to select more than one.')

        PLACE_CHOICES = [(str(place.id), place.name) for place in places]
        self.fields['place'].choices = PLACE_CHOICES

        if request.LANGUAGE_CODE == 'en':
            self.fields['date'].widget = DatePickerInput(format='%Y-%m-%d')
        elif request.LANGUAGE_CODE == 'ru' or request.LANGUAGE_CODE == 'uk':
            self.fields['date'].widget = DatePickerInput(format='%d.%m.%Y')

class PhotosForm(forms.ModelForm):
    """ Photos Form """
    image = forms.ImageField(
        label=_('Image'),
        validators=[
            validators.FileExtensionValidator(
                allowed_extensions=('jpg', 'png'))],
        error_messages={
            'invalid_extension': _('This file format is not supported.')},
        widget=forms.widgets.ClearableFileInput(attrs={'multiple': True}),
        required=False)

    class Meta:
        model = Photo
        fields = ['image']

class VideoReportForm(forms.ModelForm):
    """ VideoReport Form """
    image = forms.ImageField(
        label=_('Image'),
        help_text=_('The minimum image size should be 477x373!'),
        validators=[
            validators.FileExtensionValidator(
                allowed_extensions=('jpg', 'png'))],
        error_messages={
            'invalid_extension': _('This file format is not supported.')})

    class Meta:
        model = VideoReport
        fields = [
            'author',
            'place',
            'date',
            'title',
            'pay_to_photographer',
            'pay_to_portal',
            'image',
            'url_address',
        ]

    def __init__(self, request, *args, **kwargs):
        super(VideoReportForm, self).__init__(*args, **kwargs)
        region = get_object_or_404(Region, slug=request.session['user_region'])
        authors = User.objects.filter(
            user_access__region=region, user_access__is_photographer=True)
        places = Place.objects.filter(region=region, is_closed=False)

        AUTHOR_CHOICES = []
        for author in authors:
            if author.get_full_name():
                AUTHOR_CHOICES.append((str(author.id), author.get_full_name()))
            else:
                AUTHOR_CHOICES.append((str(author.id), author.get_username()))
        self.fields['author'].choices = AUTHOR_CHOICES

        PLACE_CHOICES = [(str(place.id), place.name) for place in places]
        self.fields['place'].choices = PLACE_CHOICES

        if request.LANGUAGE_CODE == 'en':
            self.fields['date'].widget = DatePickerInput(format='%Y-%m-%d')
        elif request.LANGUAGE_CODE == 'ru' or request.LANGUAGE_CODE == 'uk':
            self.fields['date'].widget = DatePickerInput(format='%d.%m.%Y')


#!/usr/bin/python3

import argparse
import csv
import json
import logging
import os
import sqlite3 as sl
import sys
import time
import urllib.parse
from datetime import datetime
from pathlib import Path

import requests
from __init__ import __prog__, __version__
from dotenv import load_dotenv
from rich.console import Console
from rich.progress import track

# Get the current timestamp
now = datetime.now()
dt_formatted = now.strftime("%d/%m/%Y %H:%M:%S")

# Create filename to save messages if provider is down
# Format YYYY-MM-DD
dt_formatted_filename = now.strftime("%Y-%m-%d")

# Get Process ID
pid = os.getpid()

# Construct filename to save message state
filename = f"ratatoskr_{dt_formatted_filename}_{pid}.json"

# Create a custom logger
logger = logging.getLogger(__name__)

# Create handlers
c_handler = logging.StreamHandler()
f_handler = logging.FileHandler(f"ratatoskr_{dt_formatted_filename}_{pid}.log")
c_handler.setLevel(logging.INFO)
f_handler.setLevel(logging.INFO)

# Create formatters and add it to handlers
c_format = logging.Formatter("%(name)s - %(levelname)s - %(message)s")
f_format = logging.Formatter("%(asctime)s - %(name)s - %(levelname)s - %(message)s")
c_handler.setFormatter(c_format)
f_handler.setFormatter(f_format)

# Add handlers to the logger
logger.addHandler(c_handler)
logger.addHandler(f_handler)

# Define header values
USERAGENT = f"ratatoskr-{__version__}"

# Init rich console
console = Console()

# Load .env file
load_dotenv()

def verify_environment(environment_variable):
    """Verify if a provided environment variable has a value and return that value if true"""

    value = os.getenv(environment_variable)
    if not value:
        logger.error(f"No environment variable defined for {environment_variable}")
        return None
    else:
        logger.info(
            f"Identified environment variable is present {environment_variable}"
        )
        return value

def verify_gitlab_token(session):
    """Check API if access token is valid

    session - Requests session object with headers applied for GitLab token auth
    """

    query_url = "https://gitlab.com/api/v4/personal_access_tokens"
    response = session.get(query_url, timeout=5)
    response_json = response.json()

    # Check if active
    if response.ok:
        logger.info(f"Verified GitLab Token is active")
        return True

    # If we don't have 2XX status code
    if not response.ok:
        # Check if we have expired GitLab Token
        if response_json["message"] == "401 Unauthorized":
            console.print(
                f"[!] Error - Unauthorized, verify GitLab token!",
                style="bold red",
            )
            logger.error(f"Invalid token for GitLab!")
            return None

def verify_github_token(session):
    """Check API if access token is valid

    session - Requests session object with headers applied for GitHub token auth
    """
    query_url = "https://api.github.com/user"
    response = session.get(query_url, timeout=5)
    response_json = response.json()

    # Check if we have expired GitHub Token
    if response.status_code == 401:
        console.print(
            f"[!] Error - Unauthorized, verify GitHub token is accurate and not expired! {response_json['message']}",
            style="bold red",
        )
        logger.error(f"Invalid token for GitHub!")
        return None

    # Check if active
    if response.ok:
        logger.info(f"Verified GitHub Token is active")
        return True

def get_ratelimit_status(session):
    """Get the rate limit status from GitHub API"""

    query_url = "https://api.github.com/rate_limit"
    response = session.get(query_url, timeout=5)

    if response.status_code == 200:
        requests_remaining = int(response.headers["X-RateLimit-Remaining"])
        requests_reset_time = int(response.headers["X-RateLimit-Reset"])
        result = (requests_remaining, requests_reset_time)
        return result
    if response.status_code != 200:
        return None

def get_urls(filename):
    """Read in list of GitHub repositories to monitor releases"""

    # We use a list of tuples instead of a dictionary since the owners (or keys)
    # may be duplicates for different repos by same owner in URL list
    repositories = []

    with open(filename) as csv_file:
        csv_reader = csv.reader(csv_file, delimiter=",")
        line_count = 0
        for row in csv_reader:
            if line_count == 0:
                line_count += 1
                pass
            else:
                components = row[0].split("/")
                owner = components[3]
                repo = components[4]
                if "gitlab" in row[0]:
                    combo = (owner, repo, "gitlab")
                    repositories.append(combo)
                    line_count += 1
                elif "github" in row[0]:
                    combo = (owner, repo, "github")
                    repositories.append(combo)
                    line_count += 1
                else:
                    line_count += 1
        console.print(f"[*] INFO - Processed {line_count} lines.", style="bold green")
        return repositories

def get_gitlab_latest_release(session, repository):
    """Get latest release for given GitLab public project

    session - A requests Session object
    repository - A tuple like ('Owner', 'Repo', 'gitlab')
    """

    # GitLab API will take URL-encoded namespace/project URI component
    project = urllib.parse.quote_plus(f"{repository[0]}/{repository[1]}")
    query_url = f"https://gitlab.com/api/v4/projects/{project}/releases"
    response = session.get(query_url, timeout=5)

    if "json" in response.headers.get("Content-Type"):
        response_json = response.json()

        if response_json == [] and response.status_code == 200:
            console.print(
                f"\n[!] INFO - No release found for project {repository[0]}/{repository[1]}",
                style="bold yellow",
            )
            return None

    if response.status_code == 404:
        console.print(
            f"[!] WARN - Project {repository[0]}/{repository[1]} was not found at {query_url} be sure to confirm the URL",
            style="bold red",
        )

    try:
        latest_release = response_json[0]["_links"].get("self")
    except KeyError:
        return None

    if latest_release:
        return latest_release

def get_gitlab_latest_commit(session, repository):
    """Get latest commit for given GitLab public project

    session - A requests Session object
    repository - A tuple like ('Owner', 'Repo', 'gitlab')
    """

    # GitLab API will take URL-encoded namespace/project URI component
    project = urllib.parse.quote_plus(f"{repository[0]}/{repository[1]}")
    query_url = f"https://gitlab.com/api/v4/projects/{project}/repository/commits"
    response = session.get(query_url, timeout=5)

    # Check if JSON
    if "json" in response.headers.get("Content-Type"):
        response_json = response.json()

        # Check if project does not exist
        if response.status_code == 404:
            console.print(
                f"[!] WARN - Project {repository[0]}/{repository[1]} was not found at {query_url} be sure to confirm the URL",
                style="bold red",
            )

        # Check if we have expired GitLab Token
        if response.status_code == 401:
            console.print(
                f"[!] Error - Your GitLab token is expired! {response_json}",
                style="bold red",
            )
            return None

        latest_commit = response_json[0]

        # Return GitLab latest commit URL
        if latest_commit.get("web_url"):
            return latest_commit["web_url"]
    else:
        return None

def get_latest_release(session, repository):
    """Get the latest GitHub release for given repository

    session - A requests Session object
    repository - A tuple like ('Owner', 'Repo', 'github')
    """

    # Sample input ('outflanknl', 'RedELK', 'github')

    query_url = (
        f"https://api.github.com/repos/{repository[0]}/{repository[1]}/releases/latest"
    )
    response = session.get(query_url, timeout=5)

    if "json" in response.headers.get("Content-Type"):
        response_json = response.json()
        if response_json.get("html_url"):
            return response_json["html_url"]
    else:
        return None

def get_latest_commit(session, repository):
    """Get the latest commit for a given list of repos in ('Owner', 'Repo') format"""

    query_url = f"https://api.github.com/repos/{repository[0]}/{repository[1]}/commits"
    response = session.get(query_url, timeout=5)

    if not response:
        return None

    if "json" in response.headers.get("Content-Type"):
        response_json = response.json()
        latest_commit = response_json[0]
        if latest_commit.get("html_url"):
            return latest_commit["html_url"]
    else:
        return None

def update_tracker(connection, update):
    """Update tracker DB with the latest release and commit

    connection - SQLite DB Connection Object
    update - List of commit, release, datetime, GitHub username, repo name, and whether GitLab or GitHub
    """

    # update_object = [commit, release, dt_formatted, repo[0], repo[1], repo[2]]
    # sample SQL entry = its-a-feature|Mythic|0|https://github.com/its-a-feature/Mythic/commit/75a46ef1c06e58ffaed2b036c3e4adf67b72bbc4|12/05/2021 12:37:32|github

    sql = "UPDATE repo SET latest_commit = ?, latest_release = ?, last_updated = ? WHERE  owner = ? AND repo = ? AND website = ?"

    try:
        cursor = connection.cursor()
        with connection:
            cursor.execute(sql, update)
    except sl.IntegrityError as e:
        console.print(
            f"[!] ERROR - Unable to update repo {update[4]} in tracker DB",
            style="bold red",
        )
        console.print(f"{e}")
        sys.exit(1)

def insert_repo(connection, newrepo):
    """Insert newly identified repository to track

    connection - SQLite DB Connection Object
    newrepo - List of user, repo URL, last updated timestamp, GitHub or GitLab
    """

    # newrepo = [repo[0], repo[1], dt_formatted, repo[2]]

    sql = "insert into repo (owner, repo, last_updated, website) values(?, ?, ?, ?)"

    try:
        cursor = connection.cursor()
        with connection:
            cursor.execute(sql, newrepo)
    except sl.IntegrityError as e:
        console.print(
            f"[!] ERROR - Unable to insert new repo into tracker DB", style="bold red"
        )
        console.print(f"{e}")
        sys.exit(1)

def confirm_table(connection):
    """Verify if repo table has been created

    connection - SQLite DB Connection Object
    """

    cursor = connection.cursor()
    with connection:
        cursor.execute("select * FROM sqlite_master WHERE type='table' and name='repo'")
        data = cursor.fetchall()
        if len(data) == 0:
            return None
        else:
            console.print(f"[+] INFO - Table already exists", style="bold green")
            return True

def delete_repo(connection, repo):
    """Delete repository from tracker db

    connection - SQLite DB Connection Object
    repo - List of owner and repo
    """

    sql = "DELETE FROM repo WHERE owner = ? AND repo = ?"

    try:
        cursor = connection.cursor()
        with connection:
            cursor.execute(sql, repo)
    except sl.IntegrityError as e:
        console.print(
            f"[!] ERROR - Unable to delete repo from tracker DB", style="bold red"
        )
        console.print(f"{e}")
        sys.exit(1)

def confirm_repo(connection, repo):
    """Verify if the owner and repository name is already setup in the tracker database

    connection - SQLite DB Connection Object
    repo - List of owner and repo
    """

    cursor = connection.cursor()
    with connection:
        cursor.execute(
            "select * from repo WHERE owner = ? AND repo = ?",
            [repo[0], repo[1]],
        )
        data = cursor.fetchall()
        if len(data) == 0:
            return None
        else:
            return True

def bootstrap_db(connection):
    """Bootstrap sqlite3 db with REPO table

    connection - SQLite DB Connection Object
    """

    try:
        cursor = connection.cursor()
        with connection:
            cursor.execute(
                "create table repo (owner, repo, latest_release, latest_commit, last_updated, website)"
            )
    except sl.IntegrityError as e:
        console.print(f"[!] ERROR - Unable to create repo table", style="bold red")
        console.print(f"{e}")
        sys.exit(1)

def dump_table(connection):
    """Print the tracker database

    connection - SQLite DB Connection Object
    """

    cursor = connection.cursor()
    with connection:
        data = cursor.execute("select * from repo")
        for row in data:
            print(row)

def read_repositories(connection):
    """Return all repositories in the tracker database

    connection - SQLite DB Connection Object
    """

    repositories = []

    cursor = connection.cursor()
    with connection:
        data = cursor.execute("select * from repo")
        for row in data:
            repositories.append(row)
    return repositories

def save_messages(data, filename):
    """Write messages as JSON to disk in the event webhook is unsuccessful

    data - JSON object
    filename - Filname to write to disk
    """

    try:
        with open(filename, "rt") as fh:
            existing_data = json.load(fh)
    except FileNotFoundError as notfound:
        console.print(
            f"[!] WARN - Filename [blue]{filename}[/blue] does not exist [i]will create[/i]: {notfound}",
            style="bold yellow",
        )
        existing_data = []
    except IOError as e:
        console.print(
            f"[!] ERROR - Unable to read file [blue]{filename}[/blue]: {e}",
            style="bold yellow",
        )
        existing_data = []

    # Update the dict object with new data passed to function
    existing_data.append(data)

    # Write our merged JSON object to disk in the event we have to resend the messages
    with open(filename, "wt") as fh:
        json.dump(existing_data, fh)

    console.print(
        f"[!] WARN - Wrote messages to file [blue]{filename}[/blue]",
        style="bold yellow",
    )

def send_webhook(message, webhook_url, provider, filename):
    """Send web request to webhook URL

    message - Message text for chat notification
    webhook_url - Webhook URL to send request to
    provider - Webhook provider to format message
    filename - Filename to write to disk in event of failure
    """

    # https://docs.microsoft.com/en-us/microsoftteams/platform/webhooks-and-connectors/how-to/add-incoming-webhook
    if provider == "msteams":
        data = {"Text": message}

    # https://api.slack.com/messaging/webhooks
    if provider == "slack":
        data = {"text": message}

    # https://discord.com/developers/docs/resources/webhook
    if provider == "discord":
        data = {"content": message}

    # https://docs.rocket.chat/guides/administration/admin-panel/integrations
    if provider == "rocketchat":
        data = {
            "emoji": ":chipmunk:",
            "attachments": [
                {"title": "ratatoskr notify", "text": message, "color": "#764FA5"}
            ],
        }

    # HTTP POST to our Webhook URL
    r = requests.post(webhook_url, json=data)

    # Verify successful status code via .ok method on response object
    # Most APIs return 200, Discord returns 204
    if not r.ok:
        console.print(
            f"\n[!] ERROR - Webhook was unsuccessful status code is {r.status_code}: {r.text}",
            style="bold red",
        )
        # Save messages to disk in event we don't succesfully POST
        save_messages(message, filename)

        # Return condition and response object
        return (False, r)

    if r.ok:
        return (True, r)

def parse_arguments():
    """Parse command-line arguments"""

    # Define parser
    parser = argparse.ArgumentParser()

    # Define main group
    arg_group_1 = parser.add_argument_group()
    arg_group_1.add_argument(
        "-p",
        "--provider",
        type=str,
        choices=["rocketchat", "discord", "msteams", "slack"],
        help="provider to use; required with --check",
    )
    arg_group_1.add_argument(
        "-c",
        "--check",
        action="store_true",
        help="check for new repository releases and commits",
    )

    # Create mutually exclusive group
    arg_group_2 = arg_group_1.add_mutually_exclusive_group()
    arg_group_2.add_argument(
        "-l",
        "--load",
        action="store_true",
        help="load the repositories to watch into the database",
    )
    arg_group_2.add_argument(
        "-v", "--version", action="version", version=f"{__prog__} {__version__}"
    )
    arg_group_2.add_argument(
        "-e",
        "--examples",
        action="store_true",
        help="display usage examples and exit",
    )

    # Parse our arguments into internal variables
    args = parser.parse_args()

    # Cleaner variable names
    load = args.load
    check = args.check
    provider = args.provider
    examples = args.examples

    # Display examples if user requested
    if examples:
        console.print(f"Check latest commits and releases and notify Microsoft Teams")
        console.print(
            f"    {__prog__} [cyan]--check[/cyan] [cyan]--provider[/cyan] msteams",
            style="bold blue",
        )
        console.print(f"Check latest commits and releases and notify Discord")
        console.print(
            f"    {__prog__} [cyan]--check[/cyan] [cyan]--provider[/cyan] discord",
            style="bold blue",
        )
        console.print(f"Load the latest reference CSV into tracker database")
        console.print(f"    {__prog__} [cyan]--load[/cyan]", style="bold blue")
        sys.exit(0)

    # Check and webhook provider are required to format the payload
    if check and provider is None:
        console.print(
            f"[!] ERROR - Chat provider was not provided by [green]--provider[/green] argument!",
            style="bold red",
        )
        sys.exit(1)

    return {"Load": load, "Check": check, "Provider": provider}

def prepare_database(filename):
    """Prepare the database

    filename - SQLite database filename to track repos
    """

    # Check if tracker.db file exists or not
    p = Path(filename)

    if p.exists():
        console.print(f"[+] INFO - [blue]{filename}[/blue] exists", style="bold green")
        # DB Connection
        con = sl.connect(filename, timeout=5)

        # Confirm the table exists, use the connection object
        confirm_result = confirm_table(con)

        # If the table already exists in the tracker.db file
        if confirm_result:
            console.print(
                f"[+] INFO - Tracker database is already prepared", style="bold green"
            )
            return (True, con)

        # If the table does not exist, create it!
        if not confirm_result:
            console.print(
                f"[+] INFO - Preparing database tables in [blue]{filename}[/blue] file..",
                style="bold green",
            )

            # We want to try and handle the sqlite OperationalError condition at least
            try:
                bootstrap_db(con)
                return (True, con)
            except sl.OperationalError as e:
                console.print(
                    f"[!] ERROR - Database has already been initialized!",
                    style="bold red",
                )
                console.print(f"{e}")
                return (False, con)

    # If the file does not exist, create it and prepare it
    if not p.exists():
        console.print(
            f"[!] WARN - [blue]{filename}[/blue] does not exist, creating..",
            style="bold yellow",
        )
        # DB Connection
        con = sl.connect(filename, timeout=5)

        # We want to try and handle the sqlite OperationalError condition at least
        try:
            bootstrap_db(con)
            return (True, con)
        except sl.OperationalError as e:
            console.print(
                f"[!] ERROR - Database has already been initialized!",
                style="bold red",
            )
            console.print(f"{e}")
            return (False, con)

def main():
    """Main function"""

    # Print a pretty header to console
    console.print(f" :chipmunk:  ~ Ratatoskr the Norse Squirrel God ~ :chipmunk:")

    # High-level function to parse arguments
    arguments = parse_arguments()

    # Verify tokens and webhook
    github_token = verify_environment("GITHUB_TOKEN")
    gitlab_token = verify_environment("GITLAB_TOKEN")

    # Exit if we don't have GitHub API token
    if not github_token:
        console.print(
            f"[!] ERROR - No GitHub Personal Access Token in environment variables",
            style="bold red",
        )
        sys.exit(1)

    # Exit if we don't have GitLab API Token
    if not gitlab_token:
        console.print(
            f"[!] ERROR - No GitLab Personal Access Token in environment variables",
            style="bold red",
        )
        sys.exit(1)

    # Parse provider and format for env check
    if arguments["Provider"]:
        prefix = arguments["Provider"].upper()

        # Verify that our provider webhook is in the environment
        webhook_url = verify_environment(f"{prefix}_WEBHOOK")

        # Exit if we don't have webhook URL
        if not webhook_url:
            console.print(
                f"[!] ERROR - No webhook URL found in environment variables",
                style="bold red",
            )
            sys.exit(1)

    # Define our headers
    github_custom_headers = {
        "Accept": "application/vnd.github.v3+json",
        "User-Agent": USERAGENT,
        "Authorization": "token {}".format(github_token),
    }
    gitlab_custom_headers = {
        "Accept": "application/vnd.github.v3+json",
        "User-Agent": USERAGENT,
        "PRIVATE-TOKEN": "{}".format(gitlab_token),
    }

    # Create reusable HTTP session object
    s_github = requests.Session()
    s_gitlab = requests.Session()

    # Update Session headers
    s_github.headers.update(github_custom_headers)
    s_gitlab.headers.update(gitlab_custom_headers)

    # Prepare the tracker.db file before loading data
    db_prep_result = prepare_database("tracker.db")

    # Ensure we got True from previous function call
    if not db_prep_result[0]:
        console.print(f"[!] ERROR - Preparing database!", style="bold red")
        sys.exit(1)

    # Use a friendly name for our connection object
    db_connection_handler = db_prep_result[1]

    # Check rate limits
    github_ratelimit_response = get_ratelimit_status(s_github)

    # Check GitHub token validity
    result = verify_github_token(s_github)

    # Check GitLab token validity
    result = verify_gitlab_token(s_gitlab)

    if github_ratelimit_response is None:
        console.print(f"[!] ERROR Unable to confirm rate limits", style="bold red")
        sys.exit(1)

    # If user provided --load argument, read CSV and load into tracker
    if arguments["Load"]:
        # Extract all the URLs from the first column in the CSV
        repositories = get_urls("GitHub_Tools_List.csv")

        console.print(
            f"[+] Loading repositories to monitor into tracker..", style="bold green"
        )

        # We enumerate over all repository URLs
        for repo in track(
            sequence=repositories,
            description="Loading...",
            update_period=1.0,
            auto_refresh=False,
        ):
            # Check if already tracking in database
            confirmation = confirm_repo(db_connection_handler, repo)

            # Verify that repository has been inserted into tracker
            if confirmation:
                pass
            # If it has not been inserted, load it in
            elif not confirmation:
                # Prepare our SQL insert
                newrepo = [repo[0], repo[1], dt_formatted, repo[2]]
                # Perform insert by passing our DB handler and new repository
                insert_repo(db_connection_handler, newrepo)
        sys.exit(0)

    # If user provided --check argument, read all repos from DB
    if arguments["Check"]:
        # Read tracker.db and populate all our repositories in memory
        repositories = read_repositories(db_connection_handler)

        for count, repo in enumerate(
            track(
                sequence=repositories,
                description="Checking repositories...",
                update_period=1.0,
            )
        ):
            if repo[5] == "github":
                # Get latest release URL
                release = get_latest_release(s_github, repo)
                # Get latest commit URL
                commit = get_latest_commit(s_github, repo)
            if repo[5] == "gitlab":
                # Get latest release URL
                release = get_gitlab_latest_release(s_gitlab, repo)
                # Get latest commit URL
                commit = get_gitlab_latest_commit(s_gitlab, repo)

            # Check if latest release matches DB
            if repo[2] != release and release is not None:
                console.print(
                    f"\n[+] New release for repository {repo[1]}: {release}",
                    style="bold green",
                )

                # Update the database
                update = [commit, release, dt_formatted, repo[0], repo[1], repo[5]]
                update_tracker(db_connection_handler, update)

                # Send notification to webhook
                message = f"New release for repository {repo[1]}: {release}"
                response = send_webhook(
                    message, webhook_url, arguments["Provider"], filename
                )

                # If response code is 429, backoff
                if response[1].status_code == 429:
                    delay_time = 60
                    console.print(
                        f"[!] WARN - Too many requests, backing off for [blue]{delay_time}[/blue] seconds"
                    )
                    time.sleep(delay_time)
                    response = send_webhook(
                        message, webhook_url, arguments["Provider"], filename
                    )

            if repo[3] != commit and commit is not None:
                console.print(
                    f"\n[+] New commit for repository {repo[1]}: {commit}",
                    style="bold green",
                )

                # Update the database
                update = [commit, release, dt_formatted, repo[0], repo[1], repo[5]]
                update_tracker(db_connection_handler, update)

                # Send notification to webhook
                message = f"New commit for repository {repo[1]}: {commit}"
                response = send_webhook(
                    message, webhook_url, arguments["Provider"], filename
                )

                # If response code is 429, backoff
                if response[1].status_code == 429:
                    delay_time = 60
                    console.print(
                        f"[!] WARN - Too many requests, backing off for [blue]{delay_time}[/blue] seconds"
                    )
                    time.sleep(delay_time)
                    response = send_webhook(
                        message, webhook_url, arguments["Provider"], filename
                    )

if __name__ == "__main__":
    main()


import csv
import os
from typing import Dict, Generator, Iterable, Optional, Tuple

import numpy as np

from ..classification import ModuleClassifier
from ..classification.settings import CLASS_FIELD, TEXT_FIELDS

class Preprocessor:
    def __init__(self, csv_file: str):
        self.csv_file = csv_file

    def read_csv(self) -> Iterable[Dict[str, str]]:
        with open(self.csv_file) as f:
            reader = csv.DictReader(f, dialect="excel")
            for row in reader:
                if row:
                    yield row

    def write_fasttext(
        self,
        target_file: str,
        split: Tuple[float, float, float] = (1.0, 0.0, 0.0),
        suffixes=(".train", ".dev", ".test"),
        seed: Optional[int] = None,
    ):
        if not sum(split) == 1.0:
            raise ValueError("Split must sum up to 1.0")
        np.random.seed(seed)

        target_files = [target_file + suffix for suffix in suffixes]
        handlers = [open(f, "w") for f in target_files]

        for line in self.generate_fasttext_lines():
            handler = np.random.choice(handlers, p=split)
            handler.write(line)
            handler.write(os.linesep)

        for handler in handlers:
            handler.close()

    def generate_fasttext_lines(self) -> Generator[str, None, None]:
        return (
            ModuleClassifier.fasttext_line(row, TEXT_FIELDS)
            for row in self.read_csv()
            if row.get(CLASS_FIELD)
        )


from selenium import webdriver
from selenium.webdriver.common.by import By
import math

link = "http://suninjuly.github.io/redirect_accept.html"

browser = webdriver.Chrome()

try:
    print("Opening link...")
    browser.get(link)

    print("Clicking button to open a new tab...")
    button = browser.find_element(By.CSS_SELECTOR, "button.trollface")
    button.click()

    print("Switching to the new tab...")
    browser.switch_to.window(browser.window_handles[1])

    print("Finding the value of x...")
    x_element = browser.find_element(By.CSS_SELECTOR, "#input_value")
    x = int(x_element.text)
    print(f"Value of x is {x}.")

    y = str(math.log(abs(12*math.sin(x))))
    print(f"Entering the value of y ({y})...")
    input_field = browser.find_element(By.CSS_SELECTOR, "#answer")
    input_field.send_keys(y)

    print("Submitting form...")
    submit_button = browser.find_element(By.CSS_SELECTOR, "button[type='submit']")
    submit_button.click()
    print("Completed!")

finally:
    print("Closing the browser...")
    browser.quit()


'''
    FRANCESCO GERRATANA WWW.NEXTECHNICS.COM 2020
    Scraping with python Bing COVID-19.
'''
import urllib
import requests
import json
from operator import itemgetter
from tabulate import _table_formats, tabulate
import sys
import os
import subprocess

#Afghanistan
#Albania
#Algeria
#Andorra
#Angola
#Anguilla
#Antigua e Barbuda
#Arabia Saudita
#Argentina
#Armenia
#Aruba
#Australia
#Austria
#Azerbaigian
#Bahamas
#Bahrein
#Bangladesh
#Barbados
#Belgio
#Belize
#Benin
#Bermuda
#Bhutan
#Bielorussia
#Bolivia
#Bonaire
#Bosnia ed Erzegovina
#Botswana
#Brasile
#Brunei
#Bulgaria
#Burkina Faso
#Burundi
#Cabo Verde
#Cambogia
#Camerun
#Canada
#Cayman
#Cechia
#Ciad
#Cile
#Cina (continentale)
#Cipro
#Cisgiordania
#Città del Vaticano
#Colombia
#Congo
#Congo (RDC)
#Corea del Sud
#Costa Rica
#Croazia
#Cuba
#Curaçao
#Côte d’Ivoire (Costa d’Avorio)
#Danimarca
#Dominica
#Ecuador
#Egitto
#El Salvador
#Emirati Arabi Uniti
#Eritrea
#Estonia
#Etiopia
#Figi
#Filippine
#Finlandia
#Francia
#Gabon
#Gambia
#Georgia
#Germania
#Ghana
#Giamaica
#Giappone
#Gibilterra
#Gibuti
#Giordania
#Grecia
#Grenada
#Groenlandia
#Guadalupa
#Guam
#Guatemala
#Guernsey
#Guinea
#Guinea Equatoriale
#Guinea-Bissau
#Guyana
#Guyana francese
#Haiti
#Honduras
#India
#Indonesia
#Iran
#Iraq
#Irlanda
#Islanda
#Isola di Man
#Isole Falkland
#Isole Fær Øer
#Isole Marianne settentrionali
#Isole Turks e Caicos
#Isole Vergini Americane
#Isole Vergini Britanniche
#Israele
#Italia
#Jersey
#Kazakistan
#Kenya
#Kirghizistan
#Kosovo
#Kuwait
#Laos
#Lettonia
#Libano
#Liberia
#Libia
#Liechtenstein
#Lituania
#Lussemburgo
#Macedonia del Nord
#Madagascar
#Malawi
#Malaysia
#Maldive
#Mali
#Malta
#Marocco
#Martinica
#Mauritania
#Mauritius
#Mayotte
#Messico
#Moldova
#Monaco
#Mongolia
#Montenegro
#Montserrat
#Mozambico
#Myanmar
#Namibia
#Nepal
#Nicaragua
#Niger
#Nigeria
#Norvegia
#Nuova Caledonia
#Nuova Zelanda
#Oman
#Paesi Bassi
#Pakistan
#Panamá
#Papua Nuova Guinea
#Paraguay
#Perù
#Polinesia francese
#Polonia
#Portogallo
#Portorico
#Qatar
#RAS di Hong Kong
#RAS di Macao
#Regno Unito
#Repubblica Centrafricana
#Repubblica Dominicana
#Riunione
#Romania
#Ruanda
#Russia
#Saba
#Saint Lucia
#Saint Martin
#Saint-Barthélemy
#Saint-Pierre e Miquelon
#San Marino
#Senegal
#Serbia
#Seychelles
#Sierra Leone
#Singapore
#Siria
#Slovacchia
#Slovenia
#Somalia
#Spagna
#Sri Lanka
#Stati Uniti
#Sud Sudan
#Sudafrica
#Sudan
#Suriname
#Svezia
#Svizzera
#São Tomé e Príncipe
#Taiwan
#Tanzania
#Thailandia
#Timor Est
#Togo
#Trinidad e Tobago
#Tunisia
#Turchia
#Ucraina
#Uganda
#Ungheria
#Uruguay
#Uzbekistan
#Vanuatu
#Venezuela
#Vietnam
#Yemen
#Zambia
#Zimbabwe
#eSwatini

#response = requests.get("https://www.bing.com/covid/data")
#url = json.loads(response.text)

def covidscrapy(data,location):

    my_list = []
    totc, totd, totr= 0,0,0
    tc, td, tr= 0,0,0
    totp = 0
    st, sn = "",""
    somma,sommad,sommag = 0,0,0
    totpd = 0

    for each in data['areas']:
      if each['displayName'] == location:
        st = each['displayName']
        totc = each['totalConfirmed']
        totd = each['totalDeaths']
        totr = each['totalRecovered']
        totp = (each['totalConfirmed']-(each['totalDeaths']+each['totalRecovered']))
        my_list.append((st,"",totc,totd,totr,totp))
        for x in each['areas']:
           sn = x['displayName']
           tc = x['totalConfirmed']
           td = x['totalDeaths']
           tr = x['totalRecovered']

          #gestisco il valore None
           if tc is not None and td is not None and tr is not None:
                totpd = (tc-(td+tr))#si si si
           elif tc is None and td is not None and tr is not None:
                totpd = (td+tr)#no si si
           elif tc is None and td is None and tr is not None:
                totpd = (td+tr)#no no si
           elif tc is None and td is None and tr is None:
                totpd = (0)#no no no
           elif tc is None and td is not None and tr is None:
                totpd = (td)#no si no
           elif tc is not None and td is not None and tr is None:
                totpd = (tc-(td+0))#si si no
           elif tc is not None and td is None and tr is not None:
                totpd = (tc-(0+tr))#si no si

           #calcolo manuale
           if x['totalConfirmed'] is not None:
               somma = somma + x['totalConfirmed']
           if x['totalDeaths'] is not None:
               sommad = sommad + x['totalDeaths']
           if x['totalRecovered'] is not None:
               sommag = sommag + x['totalRecovered']

           my_list.append(("",sn,tc,td,tr,totpd))

        #calcolo manuale
        my_list.append(("Somma","",somma,sommad,sommag,somma-(sommad+sommag)))

    return sorted(my_list, key=itemgetter(2),reverse=True)

url = json.load(open('bing.json', encoding='utf-8'))

db = covidscrapy(url,input("Enter state: ").title())

header = ['STATO','REGIONE',' TOT CONFERMATI',' TOT DECESSI',' TOT GUARITI','TOT POSITIVI']
print(tabulate(db,header,tablefmt="grid"))

with open("file.txt", "w",encoding='utf-8') as output:
    output.write(str(tabulate(db,header,tablefmt="grid")))

subprocess.call(["cmd", "/k", "start", "", "file.txt"], stderr=subprocess.STDOUT)


import os

import torch
import pandas as pd
from transformers import BertTokenizer, BertForSequenceClassification, AdamW, TrainingArguments, Trainer
from torch.utils.data import DataLoader, TensorDataset, random_split

from transformers import BertModel
import torch.nn as nn

import wandb

class BertRegression(nn.Module):
    def __init__(self, num_outputs):
        super(BertRegression, self).__init__()
        self.bert = BertModel.from_pretrained('bert-base-uncased')
        self.regressor = nn.Linear(self.bert.config.hidden_size, num_outputs)

    def forward(self, input_ids, attention_mask):
        outputs = self.bert(input_ids=input_ids, attention_mask=attention_mask)
        pooled_output = outputs[1]
        return self.regressor(pooled_output)

use_wandb = True

# 1. Data Preparation
data = pd.read_csv('training.csv')
tweets = data['SocialMediaFeed'].tolist()
labels = data[['NVDA', 'ING', 'SAN', 'PFE', 'CSCO']].values.tolist()

device = 'cuda' if torch.cuda.is_available() else 'cpu'

# 2. Preprocess the Data
tokenizer = BertTokenizer.from_pretrained('bert-base-uncased')
encoded_data = tokenizer(tweets, truncation=True, padding=True, max_length=256, return_tensors="pt")
input_ids = encoded_data['input_ids'].to(device)
attention_masks = encoded_data['attention_mask'].to(device)
labels = torch.tensor(labels).to(device)
labels *= 100

dataset = TensorDataset(input_ids, attention_masks, labels)
train_size = int(0.8 * len(dataset))
val_size = len(dataset) - train_size
train_dataset, val_dataset = random_split(dataset, [train_size, val_size])

train_dataloader = DataLoader(train_dataset, shuffle=True, batch_size=8)
val_dataloader = DataLoader(val_dataset, batch_size=8)

# 3. Model Setup
model = BertRegression(num_outputs=5).to(device)

# 4. Training
optimizer = AdamW(model.parameters(), lr=1e-5)
loss_fn = torch.nn.MSELoss()

if use_wandb:
    wandb.init(project="stock-prediction")
    wandb.watch(model)

MODEL_DIR = 'model_checkpoints'
if not os.path.exists(MODEL_DIR):
    os.makedirs(MODEL_DIR)

epochs = 10
for epoch in range(epochs):
    model.train()
    for batch in train_dataloader:
        input_ids, attention_mask, batch_labels = batch

        outputs = model(input_ids, attention_mask=attention_mask)
        loss = loss_fn(outputs, batch_labels)
        print("Epoch:", str(epoch), "-- Training Loss:", str(loss.item()))
        if use_wandb:
            wandb.log({"Training Loss": loss.item()})

        optimizer.zero_grad()
        loss.backward()
        optimizer.step()

    # Validation
    model.eval()
    val_loss = 0
    with torch.no_grad():
        for batch in val_dataloader:
            input_ids, attention_mask, batch_labels = batch

            outputs = model(input_ids, attention_mask=attention_mask)
            loss = loss_fn(outputs, batch_labels)
            val_loss += loss.item()
            print("Epoch:", str(epoch), "-- Validation Loss:", str(loss.item()))

    print(f"Epoch: {epoch+1}, Validation Loss: {val_loss/len(val_dataloader)}")
    if use_wandb:
        wandb.log({"Validation Loss": val_loss/len(val_dataloader)})

    model_checkpoint_path = os.path.join(MODEL_DIR, f'epoch_{epoch+1}.pth')
    torch.save(model.state_dict(), model_checkpoint_path)
    if use_wandb:
        wandb.save(model_checkpoint_path)


from django.shortcuts import render,get_object_or_404,redirect
from django.utils import timezone
from .models import Plat,Good,Post,love
from .forms import goods_form,ReplyForm
from plat import models
from accounts.models import UserInfo
from django.contrib.auth.decorators import login_required
from utils.pagination import Pagination
from django.db.models import Q
from django.http import JsonResponse
from django.forms.models import model_to_dict

# Create your views here.
def all_good(request):
    count = models.Good.objects.filter().count()
    p = request.GET.get('p')
    path_info = request.META.get('PATH_INFO')
    req = Pagination(p, count, per_page_count=9)
    page_str = req.page_str(path_info)
    goods = models.Good.objects.filter().all()[req.start:req.end]

    p1 = models.Good.objects.filter(plat='书籍教材').count()
    p2 = models.Good.objects.filter(plat='电子设备').count()
    p3 = models.Good.objects.filter(plat='生活用品').count()
    p_counts = {
        'p1': p1,
        'p2': p2,
        'p3': p3,
    }

    c1 = models.UserInfo.objects.filter(Institute='信息工程学院').count()
    c2 = models.UserInfo.objects.filter(Institute='自动化学院').count()
    c3 = models.UserInfo.objects.filter(Institute='计算机学院').count()
    c4 = models.UserInfo.objects.filter(Institute='外国语学院').count()
    c5 = models.UserInfo.objects.filter(Institute='轻化学院').count()
    c6 = models.UserInfo.objects.filter(Institute='物理学院').count()
    counts = {
        'c1': c1,
        'c2': c2,
        'c3': c3,
        'c4': c4,
        'c5': c5,
        'c6': c6,
    }

    name = request.GET.get('name')
    if name:
        count = models.Good.objects.filter(good_name=name).count()
        p = request.GET.get('p')
        path_info = request.META.get('PATH_INFO')
        req = Pagination(p, count, per_page_count=9)
        page_str = req.page_str(path_info)
        goods = models.Good.objects.filter(good_name=name).all()[req.start:req.end]

    plat = request.GET.get('plat')
    if plat:
        count = models.Good.objects.filter(plat=plat).count()
        p = request.GET.get('p')
        path_info = request.META.get('PATH_INFO')
        req = Pagination(p, count, per_page_count=9)
        page_str = req.page_str(path_info)
        goods = models.Good.objects.filter(plat=plat).all()[req.start:req.end]

    username = request.GET.get('user')
    user = UserInfo.objects.filter(username=username).first()
    if user:
        count = models.Good.objects.filter(starter=user).count()
        p = request.GET.get('p')
        path_info = request.META.get('PATH_INFO')
        req = Pagination(p, count, per_page_count=9)
        page_str = req.page_str(path_info)
        goods = models.Good.objects.filter(starter=user).all()[req.start:req.end]

    online = request.GET.get('online')
    if online:
        count = models.Good.objects.filter(online=1).count()
        p = request.GET.get('p')
        path_info = request.META.get('PATH_INFO')
        req = Pagination(p, count, per_page_count=9)
        page_str = req.page_str(path_info)
        goods = models.Good.objects.filter(online=1).all()[req.start:req.end]

    argue = request.GET.get('argue')
    if argue:
        count = models.Good.objects.filter(lower=1).count()
        p = request.GET.get('p')
        path_info = request.META.get('PATH_INFO')
        req = Pagination(p, count, per_page_count=9)
        page_str = req.page_str(path_info)
        goods = models.Good.objects.filter(lower=1).all()[req.start:req.end]

    time_order = request.GET.get('time_order')
    if time_order:
        count = models.Good.objects.filter().count()
        p = request.GET.get('p')
        path_info = request.META.get('PATH_INFO')
        req = Pagination(p, count, per_page_count=9)
        page_str = req.page_str(path_info)
        goods = models.Good.objects.filter().order_by('-last_updated').all()[req.start:req.end]

    count_order = request.GET.get('count_order')
    if count_order:
        count = models.Good.objects.filter().count()
        p = request.GET.get('p')
        path_info = request.META.get('PATH_INFO')
        req = Pagination(p, count, per_page_count=9)
        page_str = req.page_str(path_info)
        goods = models.Good.objects.filter().order_by('views').all()[req.start:req.end]

    price_order = request.GET.get('price_order')
    if price_order:
        count = models.Good.objects.filter().count()
        p = request.GET.get('p')
        path_info = request.META.get('PATH_INFO')
        req = Pagination(p, count, per_page_count=9)
        page_str = req.page_str(path_info)
        goods = models.Good.objects.filter().order_by('price').all()[req.start:req.end]

    return render(request, 'allgoods.html', {'goods': goods,"page_str": page_str,'counts':counts,'p_counts':p_counts})

@login_required
def plat_goods(request):
    user = request.user
    user = UserInfo.objects.filter(username=user).first()
    count = models.Good.objects.filter(starter=user).count()
    p = request.GET.get('p')
    path_info = request.META.get('PATH_INFO')
    req = Pagination(p, count, per_page_count=3)
    page_str = req.page_str(path_info)
    goods = models.Good.objects.filter(starter=user).all()[req.start:req.end]
    return render(request, 'goods.html',{'goods':goods,"page_str": page_str})

@login_required
def add_goods(request):
    form = goods_form(request.POST,request.FILES)
    user = request.user
    user = UserInfo.objects.filter(username=user).first()
    if request.method == "POST":
        form = goods_form(request.POST,request.FILES)
        if form.is_valid():
            data = {
            'starter': user,
        }
            data.update(form.cleaned_data)
            img = request.FILES.get('good_photo')
            v = Good.objects.create(**data)
            return redirect("/plat/my_good")
        else:
            print(form.errors)
    return render(request, 'add_bak.html', {'form':form})

@login_required
def edit_goods(request,id):
    obj = models.Good.objects.filter(id=id).first()
    form = goods_form(instance=obj)
    if request.method == "POST":
        form = goods_form(request.POST,request.FILES,instance=obj)
        if form.is_valid():
            img = request.FILES.get('good_photo')
            v = models.Good.objects.filter(id=id).update(**form.cleaned_data)
            return redirect("/plat/my_good")
        else:
            print(form.errors)
    return render(request, 'edit_goods.html', {'form': form,'id':id})

@login_required
def del_goods(request,id):
    goods = models.Good.objects.filter(id=id)
    goods.delete()
    return redirect("/plat/my_good")

@login_required
def good_detail(request,id):
    goods = models.Good.objects.filter(id=id)
    good = models.Good.objects.filter(id=id).first()
    post_list = models.Post.objects.filter(topic=good).all()
    username = good.starter
    user = UserInfo.objects.filter(username=username).first()
    user_goods = models.Good.objects.filter(starter=user)
    userdetail = UserInfo.objects.filter(username=username).first()
    love_user = request.user
    love_user = UserInfo.objects.filter(username=love_user).first()
    fav = models.love.objects.filter(user_id=love_user, good_id=good).first()
    good.views +=1
    good.save()

    if request.method =='POST':
        message = request.POST['message']
        good = models.Good.objects.filter(id=id).first()
        username = request.user
        userreal = UserInfo.objects.filter(username=username).first()
        post = Post()
        post.message = message
        post.updated_at = timezone.now()
        post.created_by = userreal
        post.topic = good
        post.save()
        return redirect("/plat/good_%s" % id)
    return render(request, 'good_details.html', {'goods':goods,'userdetail':userdetail,'post_list':post_list,'user_goods':user_goods,'thisgood':good,'fav':fav})

def home(request):
    plats = Plat.objects.all()
    good = models.Good.objects.filter()
    return render(request, 'index.html', {'plats': plats,'good':good})

@login_required
def loved(request,id):
    good = models.Good.objects.filter(id=id).first()
    loved= love()
    username = request.user
    user = UserInfo.objects.filter(username=username).first()
    loved.user_id = user
    loved.good_id = good
    loved.save()
    return redirect("/plat/good_%s" % id)

@login_required
def unloved(request,id):
    good = models.Good.objects.filter(id=id).first()
    username = request.user
    user = UserInfo.objects.filter(username=username).first()
    fav = models.love.objects.filter(user_id=user,good_id=good)
    fav.delete()
    return redirect("/plat/good_%s" % id)

@login_required
def love_goods(request):
    username = request.user
    userdetail = UserInfo.objects.filter(username=username).first()
    goods = models.Good.objects.filter(starter=userdetail)
    good = []
    fav = models.love.objects.filter(user_id=userdetail)
    fav_self = models.love.objects.filter(user_id=userdetail).count()
    good_self = models.Good.objects.filter(starter=userdetail).count()
    if fav:
       good = fav
    return render(request, 'love_good.html', {'userdetail':userdetail,'goods': goods,'good':good,'fav_self':fav_self,'good_self':good_self})

@login_required
def my_good(request):
    user = request.user
    user = UserInfo.objects.filter(username=user).first()
    username = request.user
    userdetail = UserInfo.objects.filter(username=username).first()
    fav_self = models.love.objects.filter(user_id=userdetail).count()
    count = models.Good.objects.filter(starter=user).count()
    p = request.GET.get('p')
    path_info = request.META.get('PATH_INFO')
    req = Pagination(p, count, per_page_count=3)
    page_str = req.page_str(path_info)
    goods = models.Good.objects.filter(starter=user).all()[req.start:req.end]
    good_self = models.Good.objects.filter(starter=userdetail).count()
    return render(request, 'my_good.html', {'userdetail':userdetail,'goods': goods,"page_str": page_str,'fav_self':fav_self,'good_self':good_self})

@login_required
def delete_post(request,id):
    post = models.Post.objects.filter(id=id).first()
    id = post.topic.id
    post.delete()
    return redirect("/plat/good_%s" % id)

def reply_post(request):
    # 二级回复
    comment_id = request.POST.get('comment_id')
    author_to_id = request.POST.get('author_to_id')
    if comment_id and author_to_id:
        try:
            comment = Post.objects.get(id=comment_id)
            author_to = UserInfo.objects.get(id=author_to_id)
            form = ReplyForm(data=request.POST)
            if form.is_valid():
                new_reply = form.save(commit=False)
                new_reply.comment = comment
                new_reply.author_from = request.user
                new_reply.author_to = author_to
                new_reply.save()
                return JsonResponse({'status': 'ok'})
        except Post.DoesNotExist or UserInfo.DoesNotExist:
            return JsonResponse({'status': 'ko'})
    return JsonResponse({'status': 'ko'})


import cv2
import os
import numpy as np

mate_color_dict = {"blue_button": [[50, 110], [150, 200], [220, 255]],
                   "white": [[240, 255], [240, 255], [240, 255]],
                   "grey": [[200, 230], [200, 230], [200, 230]],
                   "red": [[200, 255], [40, 80], [40, 80]],
                   "blue": [[90, 110], [180, 210], [230, 255]],
                   }

def is_contain(rects):
    rects = [[rect[0], rect[1], rect[0] + rect[2], rect[1] + rect[3]] for rect in rects]
    return (rects[0][0] < rects[1][0] and rects[0][1] < rects[1][1] and rects[1][2] < rects[0][2]
            and rects[1][3] < rects[0][3]) or \
           (rects[1][0] < rects[0][0] and rects[1][1] < rects[0][1] and rects[0][2] < rects[1][2]
            and rects[0][3] < rects[1][3])

def is_color(pixel, thresh):
    return thresh[2][1] >= pixel[0] >= thresh[2][0] and thresh[1][1] >= pixel[1] >= thresh[1][0] and \
           thresh[0][1] >= pixel[2] >= thresh[0][0]

def binary_thresh_pic(img, color_keys):
    height, width, channel = img.shape
    bg = np.zeros((height, width), dtype=np.uint8)
    for h in range(height):
        for w in range(width):
            for i, color_key in enumerate(color_keys):
                if is_color(img[h, w, :], mate_color_dict[color_key]):
                    bg[h, w] = 255
                    break
                if i == len(color_keys) - 1:
                    bg[h, w] = 0
    return bg

img = cv2.imread("12573.jpg")
# img = cv2.resize(img, None, None, 0.5, 0.5)
h, w, c = img.shape

star_h = h // 4
clip_h = h // 3
star_w = w * 9 // 10
clip_w = w * 3 // 5

img = img[star_h: star_h + clip_h, star_w:, :]

# img = cv2.resize(img, None, None, 0.5, 0.5)

# h, w, c = img.shape

binary_thresh = binary_thresh_pic(img, ["red", "blue"])

kernel = np.ones((3, 3), np.uint8)
dilate = cv2.dilate(binary_thresh, kernel)

cv2.imshow("thresh", dilate)

image, contours, hierarchy = cv2.findContours(dilate, cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE)

rects = [cv2.boundingRect(box) for box in contours]

for rect in rects:
    cv2.rectangle(img, (rect[0], rect[1]), (rect[0] + rect[2], rect[1] + rect[3]), (0, 0, 255), 1)

rects = sorted(rects, key=lambda x: x[0])

if len(rects) == 2 and rects[0][0] < w / 6 and rects[0][1] > h * 2 / 3 and rects[1][0] > w / 2 and rects[1][1] > h / 2 \
        and rects[0][2] > w / 4:
    print("has_finish")

cv2.imshow("img", img)
cv2.waitKey(0)


from django.conf.urls import url
from . import views

app_name = 'music'

urlpatterns = [
    # Matches /music/
    url(r'^$', views.IndexView.as_view(), name='index'),  # Default page

    # Matches /music/albumID/
    url(r'^(?P<pk>[0-9]+)/$', views.DetailView.as_view(), name='detail'),  # /music/:albumid

    # Matches /music/album/add/
    url(r'^album/add/$', views.AlbumCreate.as_view(), name='album-add'),

    # Matches /music/album/:albumID/edit/
    url(r'^album/(?P<pk>[0-9]+)/edit$', views.AlbumUpdate.as_view(), name='album-edit'),

    # Matches /music/album/albumID/delete/
    url(r'^album/(?P<pk>[0-9]+)/delete', views.AlbumDelete.as_view(), name='album-delete'),

]


import os
from openpyxl import load_workbook
import re
from nltk.stem.wordnet import WordNetLemmatizer
from nltk.corpus import stopwords
from itertools import groupby

special_words = ['prosses', 'acsses']

class DataLoader():
    def __init__(self, wb, sheetname):
        self.sheetname = sheetname
        self.sheet = wb[self.sheetname]
        self.questions, self.categories, id_cate_dict = self._get_questions()

    def _get_questions(self):
        m_row = self.sheet.max_row
        id_cate_dict = []
        questions = []
        categories = []
        for i in range(2, m_row + 1):
            cell_obj = self.sheet.cell(row = i, column = 2)
            if not cell_obj.value:
                continue
            questions.append(cell_obj.value)
            categories.append(self.sheet.cell(row=i, column=6).value)
            try:
                id_cate_dict.append((self.sheet.cell(row=i, column=1).value.split('/')[-1].strip(), self.sheet.cell(row=i, column=6).value.strip().replace('\n', ' ')))
            except:
                print(i)
        return questions, categories, id_cate_dict

    def _check_pattern(self, token):
        method_patterns = [re.compile('\w+\.+'), re.compile('\w+\_+')]
        for method_pattern in method_patterns:
            if re.search(method_pattern, token):
                return '<method>'
            elif re.search('\_', token):
                return '<method>'
            else:
                return token

    def _process_ques(self):
        new_ques = []
        stop_words = set(stopwords.words('english'))
        for idx, line in enumerate(self.questions):
            if not line:
                continue
            terms = line.lower().split()
            for term_id, term in enumerate(terms):
                terms[term_id] = re.sub('[\"\(\)\`\=\:\[\]\-\,\?\+\/]*', ' ',term)
                if re.search(re.compile("(\w+)\'s"), terms[term_id]):
                    terms[term_id] = terms[term_id].split("'")[0]
                terms[term_id] = re.sub("[\']*", '', terms[term_id])
                terms[term_id] = re.sub(' +', '', terms[term_id])
                terms[term_id] = self._check_pattern(terms[term_id])
                terms[term_id] = WordNetLemmatizer().lemmatize(terms[term_id], 'v')
                if terms[term_id] not in special_words:
                    temp_token = WordNetLemmatizer().lemmatize(terms[term_id], 'n')
                    if terms[term_id] != temp_token and not re.search(r'ss$', terms[term_id]):
                        terms[term_id] = temp_token
                if terms[term_id] in stop_words or len(terms[term_id])==1:
                    terms[term_id] = ''
                if re.match(re.compile("(\d+)d$"), terms[term_id]):   # $ is the end of string
                    terms[term_id] = '<dim>'
                elif re.match(re.compile("(\d+)gb$"), terms[term_id]):   # $ is the end of string
                    terms[term_id] = '<mem>'
                elif terms[term_id].isdigit():
                    terms[term_id] = '<digit>'

            new_terms = [x[0] for x in groupby(terms)]
            new_terms = ' '.join(new_terms)
            new_terms = re.sub(r"[^0-9a-z,.?!<>' ]", ' ', new_terms)
            new_terms = re.sub(' +', ' ', new_terms)
            category = self.categories[idx].strip().replace('\n', ' ')
            new_ques.append(new_terms.strip()+'\t'+category+'\n')
        return new_ques

def save_ques(data, fn):
    fw = open(os.path.join('..', 'data', fn), 'w')
    for item in data:
        try:
            fw.write(item)
        except TypeError:
            print(item)
            continue
    fw.close()
    print('%s has finished writing...'%fn)

# wb = load_workbook(filename=os.path.join('..', 'data', 'dl_framework_stackoverflow_questions.xlsx'))
# tf_sheet = wb['TensorFlow(sample)']
# pt_sheet = wb['PyTorch(sample)']
# dj_sheet = wb['DeepLearning4J(sample)']
#
# tf_questions = DataLoader(wb, 'TensorFlow(sample)')._process_ques()
# pt_questions = DataLoader(wb, 'PyTorch(sample)')._process_ques()
# dj_questions = DataLoader(wb, 'DeepLearning4J(sample)')._process_ques()
#
# save_ques(tf_questions, 'tensforflow.txt')
# save_ques(dj_questions, 'dl4j.txt')
# save_ques(pt_questions, 'pytorch.txt')
# save_ques(tf_questions+pt_questions+dj_questions, 'total.txt')


"""
@author: bonniema

This code helps you get relevant information from each job card, and then iterate through all job cards on one page.
Then go to the next page
"""

from selenium.common.exceptions import NoSuchElementException, ElementClickInterceptedException
from selenium import webdriver
import time
import pandas as pd

options = webdriver.ChromeOptions()

#Uncomment the line below if you'd like to scrape without a new Chrome window every time.
#options.add_argument('headless')

#Change the path to where chromedriver is in your home folder.
#driver = webdriver.Chrome(executable_path="D:\job_applications-2020\DEC_JAN_2021\SharpestMinds\ds_salary_proj/chromedriver", options=options)
driver = webdriver.Chrome(executable_path=path, options=options)
driver.set_window_size(1120, 1000)
#let the driver wait 3 seconds to locate the element before exiting out
driver.implicitly_wait(3)

titles=[]
companies=[]
locations=[]
links =[]
reviews=[]
salaries = []
descriptions=[]

for i in range(0,20):

    job_card = driver.find_elements_by_xpath('//div[contains(@class,"clickcard")]')

    for job in job_card:

    #.  not all companies have review
        try:
            review = job.find_element_by_xpath('.//span[@class="ratingsContent"]').text
        except:
            review = "None"
        reviews.append(review)
   #.   not all positions have salary
        try:
            salary = job.find_element_by_xpath('.//span[@class="salaryText"]').text
        except:
            salary = "None"
    #.  tells only to look at the element
        salaries.append(salary)

        try:
            location = job.find_element_by_xpath('.//span[contains(@class,"location")]').text
        except:
            location = "None"
    #.  tells only to look at the element
        locations.append(location)

        try:
            title  = job.find_element_by_xpath('.//h2[@class="title"]//a').text
        except:
            title = job.find_element_by_xpath('.//h2[@class="title"]//a').get_attribute(name="title")
        titles.append(title)
        links.append(job.find_element_by_xpath('.//h2[@class="title"]//a').get_attribute(name="href"))
        companies.append(job.find_element_by_xpath('.//span[@class="company"]').text)

    try:
        next_page = driver.find_element_by_xpath('//a[@aria-label={}]//span[@class="pn"]'.format(i+2))
        next_page.click()

    except:
        next_page = driver.find_element_by_xpath('//a[@aria-label="Next"]//span[@class="np"]')
        next_page.click()
    #except:
        #next_page = driver.find_element_by_xpath('//a[.//span[contains(text(),"Next")]]')
        #next_page.click()

    print("Page: {}".format(str(i+2)))


import os
import argparse
import spack.main
import spack.environment as ev
import snapshot_creator

manager = spack.main.SpackCommand('manager')
concretize = spack.main.SpackCommand('concretize')
config = spack.main.SpackCommand('config')
install = spack.main.SpackCommand('install')

def command(command, *args):
    print('spack', command.command_name, *args)
    print(command(*args, fail_on_error=False))

parser = argparse.ArgumentParser(
    'Run a test of the developer snapshot workflow')
parser.add_argument('--view_name', required=False,
                    help='view to use for the developer')
parser.add_argument('--snap_name', required=False,
                    help='directory name for the snapshot')
parser.add_argument('--dev_name', required=False,
                    help='directory name for the developer env')
parser.add_argument('--num_threads', required=False,
                    help='DAG paralleization')
parser.set_defaults(view_name='default', snap_name='test',
                    dev_name='test_external', num_threads=1)
args = parser.parse_args()

# set up the snapshot
print('Create Snapshot')
args = snapshot_creator.parse(
    ['--use_develop', '--modules', '--name', args.snap_name,
     '--num_threads', args.num_threads])
snapshot_path = snapshot_creator.create_snapshots(args)
print('Snapshot created at', snapshot_path)

# set up the user environment
ev.deactivate()
env_path = os.path.join(os.environ['SPACK_MANAGER'],
                        'environments', args.dev_name)
command(manager, 'create-env', '--directory', env_path, '--spec', 'nalu-wind')
ev.activate(ev.Environment(env_path))
command(manager, 'external', snapshot_path, '-v', args.view_name,
        '--blacklist', 'nalu-wind')
command(manager, 'develop', 'nalu-wind@master')
command(concretize)
command(install)


from flask import Flask, request
import ssl

app = Flask(__name__)

@app.route('/')
def hello():
    return 'Hello, world!'

if __name__ == '__main__':
    # Load server certificate and private key
    server_cert = r'C:\Users\Lenovo\Desktop\MTLS Connection\MTLS\certificate.pem'
    server_key = r'C:\Users\Lenovo\Desktop\MTLS Connection\MTLS\key.pem'

    # Create SSL context with self-signed certificate and private key
    context = ssl.create_default_context(ssl.Purpose.CLIENT_AUTH)
    context.load_cert_chain(certfile=server_cert, keyfile=server_key)

    # Run the app with SSL/TLS configured
    app.run(ssl_context=context, host='0.0.0.0', port=443)


from typing import Callable, Optional
from addict import Dict
import torch
import torch.nn.functional as F

from gvcore.utils.types import TDataList
from gvcore.utils.misc import sharpen
from gvcore.model import GenericModule

from evaluator.segmentation import SegmentationEvaluator

class TSSegmentor(GenericModule):
    """
    Teacher-Student segmentor for semi-supervised learning.
    """

    def __init__(self, cfg, segmentor_class: Callable[..., GenericModule]):
        super(TSSegmentor, self).__init__(cfg)

        # Components ========
        self.segmentor = segmentor_class(cfg)
        self.ema_segmentor = segmentor_class(cfg)
        self.ema_segmentor.eval()

        # Initialize ========
        for param_q, param_k in zip(self.segmentor.parameters(), self.ema_segmentor.parameters()):
            param_k.data.copy_(param_q.data)
            param_k.requires_grad = False

        # Buffer ========
        for m in self.ema_segmentor.modules():
            if isinstance(m, (torch.nn.BatchNorm2d, torch.nn.SyncBatchNorm)):
                m.momentum = 0.9

        self.bn_statistics_pl = Dict()
        self.bn_statistics_eval = Dict()
        self.bn_statistics_mode = "eval"

        for name, buffer in self.ema_segmentor.named_buffers():
            self.bn_statistics_pl[name] = buffer.clone()
            self.bn_statistics_eval[name] = buffer.clone()

        # Tools ========
        self.tranining_evaluator = SegmentationEvaluator(
            num_classes=cfg.model.num_classes, distributed=cfg.num_gpus > 1, mode="window_avg", window_size=100
        )

    def to(self, device):
        super(TSSegmentor, self).to(device)
        for name, _ in self.ema_segmentor.named_buffers():
            self.bn_statistics_eval[name] = self.bn_statistics_eval[name].to(device)
            self.bn_statistics_pl[name] = self.bn_statistics_pl[name].to(device)

    def train(self, mode=True):
        self.training = mode
        self.segmentor.train(mode)
        self.ema_segmentor.eval()

    def switch_to_bn_statistics_pl(self):
        if self.bn_statistics_mode != "pl":
            for name, buffer in self.ema_segmentor.named_buffers():
                self.bn_statistics_eval[name] = buffer.clone()
                buffer.data = self.bn_statistics_pl[name]
            self.bn_statistics_mode = "pl"

    def switch_to_bn_statistics_eval(self):
        if self.bn_statistics_mode != "eval":
            for name, buffer in self.ema_segmentor.named_buffers():
                self.bn_statistics_pl[name] = buffer.clone()
                buffer.data = self.bn_statistics_eval[name]
            self.bn_statistics_mode = "eval"

    def forward_generic(self, *args, **kwargs):
        selector = kwargs.pop("selector")
        if selector == "student":
            return self.segmentor.forward_generic(*args, **kwargs)
        elif selector == "teacher":
            return self.ema_segmentor.forward_generic(*args, **kwargs)

    def forward_train(
        self,
        data_list_l_weak: TDataList,
        data_list_u_weak: TDataList,
        data_list_l_strong: TDataList,
        data_list_u_strong: TDataList,
    ):
        logits_l = self.forward_generic(data_list_l_strong, selector="student")
        # 1. Train with labeled data
        labels_l = torch.cat([data.label for data in data_list_l_weak], dim=0)
        loss_dict_l = self.get_losses(logits_l, labels_l, gt_labels=labels_l, labelled=True)

        # 2. Pseudo labeling and Consistency training with labeled and unlabeled data
        pseudo_labels_u = self._pseudo_labeling(data_list_u_weak)
        labels_u = torch.cat([data.label for data in data_list_u_weak], dim=0)
        logits_u = self.forward_generic(data_list_u_strong, selector="student")
        loss_dict_u = self.get_losses(logits_u, pseudo_labels_u, gt_labels=labels_u, labelled=False)

        loss_dict = Dict()
        loss_dict.loss_l = loss_dict_l
        loss_dict.loss_u = loss_dict_u

        # 3. Calculate mIoU of pseudo labels
        stat_dict = self._pseudo_labeling_statistics(pseudo_labels_u, labels_u)

        return loss_dict, stat_dict

    def get_losses(
        self,
        logits: torch.Tensor,
        labels: torch.Tensor,
        gt_labels: Optional[torch.Tensor] = None,
        labelled: bool = True,
    ):
        loss_dict = Dict()
        if labelled:
            loss = F.cross_entropy(logits, labels.long(), ignore_index=255)
        else:
            pseudo_probs_u, pseudo_preds_u = torch.max(labels, dim=1)
            mask = torch.ge(pseudo_probs_u, self.cfg.model.teacher.score_threshold)
            valid_mask = torch.ne(gt_labels, 255) & mask

            loss = F.cross_entropy(logits, pseudo_preds_u, reduction="none")
            loss = loss * valid_mask.float()

            loss = loss.sum() / (valid_mask.sum() + 1e-10)
        loss_dict.loss = loss

        return loss_dict

    @torch.no_grad()
    def _pseudo_labeling(self, data_list: TDataList, sharpen_factor: float = 0.5) -> torch.Tensor:
        # Update momentum batch statistic pl
        self.switch_to_bn_statistics_pl()
        self.ema_segmentor.train()
        self.forward_generic(data_list, selector="teacher")
        self.ema_segmentor.eval()

        logits = self.forward_generic(data_list, selector="teacher")
        probs = F.softmax(logits, dim=1)
        if sharpen_factor > 0:
            probs = sharpen(probs, sharpen_factor)

        self.switch_to_bn_statistics_eval()
        return probs.detach()

    @torch.no_grad()
    def _pseudo_labeling_statistics(self, pseudo_labels: torch.Tensor, gt_labels: torch.Tensor) -> Dict:
        if pseudo_labels.dim() == 4:
            pseudo_labels = torch.argmax(pseudo_labels, dim=1)

        self.tranining_evaluator.process(pseudo_labels, gt_labels)
        metrics = self.tranining_evaluator.calculate()

        return Dict(PL_mIoU=metrics["mIoU"])

    @torch.no_grad()
    def _momentum_update(self, m: float):
        """
        Momentum update of the key encoder
        """
        for buffer_q, buffer_k in zip(self.segmentor.buffers(), self.ema_segmentor.buffers()):
            buffer_k.data = buffer_k.data * m + buffer_q.data * (1.0 - m)

        for param_q, param_k in zip(self.segmentor.parameters(), self.ema_segmentor.parameters()):
            param_k.data = param_k.data * m + param_q.data * (1.0 - m)

    @torch.no_grad()
    def forward_eval(self, data_list: TDataList):
        # 1. Test student
        logits_stu = self.forward_generic(data_list, selector="student")
        probs_stu = F.softmax(logits_stu, dim=1)
        preds_stu = torch.argmax(probs_stu, dim=1)

        # 2. Test teacher
        logits_tea = self.forward_generic(data_list, selector="teacher")
        probs_tea = F.softmax(logits_tea, dim=1)
        preds_tea = torch.argmax(probs_tea, dim=1)

        return preds_stu, preds_tea


# Реализовать функцию которая возвращает дату недельной давности от текущего момента времени.
from datetime import datetime, timedelta

def date_week_before():
    today = datetime.now()
    week = timedelta(7)
    week_before = today - week
    return week_before

if __name__ == "__main__":
    print(date_week_before())


import json
import os
import re
from config_processors.silver_config_validator import SilverConfigValidator

class SilverConfig:
    def __init__(self, config_repo_path, source_system, env, territory):
        self.config_repo_path = config_repo_path
        self.source_system = source_system
        self.env = env
        self.territory = territory

    def remove_table_entries(self, table_names):
        print("Removing tables: ", table_names)

        config_dir = os.path.join(
            self.config_repo_path,
            "configs",
            self.source_system,
            self.territory,
            self.env,
        )

        app_vars_file = os.path.join(config_dir, "app_vars.json")

        if not os.path.isfile(app_vars_file):
            print(f"No such file: {app_vars_file}")
            return

        with open(app_vars_file, "r") as f:
            app_vars = json.load(f)

        table_names = set(table_names)  # convert list to set for faster operations

        # Initialize a dictionary to store removed tables
        removed_matches = {}

        for job, values in app_vars.items():
            if len(values) > 1 and isinstance(values[1], str):
                for table_name in table_names:
                    table_entry_str = (
                        f'{{ "database_name": "[^"]*", "table_name":"{table_name}" }}'
                        + ","
                    )
                    matches = re.findall(table_entry_str, values[1])
                    if matches:
                        print("\nEnrichment job::", job)
                        if job not in removed_matches:
                            removed_matches[job] = []
                        for match in matches:
                            print(f"Removing Matching record: {match}")
                        values[1] = re.sub(table_entry_str, "", values[1])

        with open(app_vars_file, "w") as f:
            json.dump(app_vars, f, indent=4)
            f.write("\n")

        self.validate_config_file(app_vars_file)

    def unpause_tables(self, table_names):
        print("Unpausing tables: ", table_names)

        metadata_file_path = os.path.join(
            "metadata", f"{self.territory}-{self.env}-app-vars.json"
        )
        app_vars_file_path = os.path.join(
            self.config_repo_path,
            "configs",
            "excite",
            self.territory,
            self.env,
            "app_vars.json",
        )
        with open(metadata_file_path, "r") as f:
            metadata = json.load(f)

        with open(app_vars_file_path, "r") as f:
            app_vars = json.load(f)

        for table_name in table_names:
            if table_name in metadata:
                job_name = metadata[table_name]["job_name"]
                value_string = metadata[table_name]["value"]

                if job_name in app_vars:
                    if table_name in app_vars[job_name][1]:
                        raise ValueError(
                            f"Error Processing::Table {table_name} already exists in {job_name}"
                        )
                    app_vars[job_name][1] = re.sub(
                        r"\]$", f", {value_string}]", app_vars[job_name][1]
                    )
                    print("\nEnrichment job::", job_name)
                    print(f"Adding table {value_string}")
                else:
                    raise ValueError(f"Job {job_name} not found in app_vars.json")
            else:
                raise ValueError(f"Table {table_name} not found in metadata")

        with open(app_vars_file_path, "w") as f:
            json.dump(app_vars, f, indent=4)
            f.write("\n")

        self.validate_config_file(app_vars_file_path)

    def validate_config_file(self, app_vars_file_path):
        validator = SilverConfigValidator()
        if not validator.validate_config_file(app_vars_file_path):
            raise ValueError(f"Invalid Config file: {app_vars_file_path}")

    def build_metadata(self):
        config_dir = os.path.join(
            self.config_repo_path,
            "configs",
            self.source_system,
            self.territory,
            self.env,
        )
        source_file_path = os.path.join(config_dir, "app_vars.json")

        if not os.path.isfile(source_file_path):
            raise FileNotFoundError(f"No such file: {source_file_path}")

        with open(source_file_path, "r") as source_file:
            data = json.load(source_file)

        metadata = {}

        # Traverse the jobs in the data
        for job_name, values in data.items():
            if len(values) > 1 and isinstance(values[1], str):
                try:
                    tables = json.loads(values[1])  # parse json string that has tables
                except json.JSONDecodeError:
                    raise ValueError(
                        f"Invalid JSON string in the value array for {job_name}"
                    )

                for table in tables:
                    table_name = table["table_name"]
                    if table_name in metadata:  # Check if table_name already exists
                        raise ValueError(f"Duplicate table name detected: {table_name}")

                    metadata[table_name] = {
                        "value": json.dumps(table, separators=(",", ":")),
                        "job_name": job_name,
                    }

        # save the metadata file
        output_file_path = os.path.join(
            "metadata", f"{self.territory}-{self.env}-app-vars.json"
        )
        with open(output_file_path, "w") as output_file:
            json.dump(metadata, output_file, indent=4)

        print(f"Metadata saved to: {output_file_path}")