#    This program is free software: you can redistribute it and/or modify
#    it under the terms of the GNU General Public License as published by
#    the Free Software Foundation, either version 3 of the License, or
#    (at your option) any later version.
#
#    This program is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#    GNU General Public License for more details.
#
#    You should have received a copy of the GNU General Public License
#    along with this program.  If not, see <http://www.gnu.org/licenses/>.

import subprocess
import re
import datetime
import logging
import io
import os.path

logging.basicConfig(level=logging.ERROR)

class DVD(object):
    def __init__(self):
        self.titles=[]
        self.name=''
        self.title=''
    def __str__(self):
        formatted="DVD filename is: "+self.name+'\n'+\
        "DVD Title is: "+self.title+'\n'
        for i in self.titles:
            formatted=formatted+str(i)+'\n'
        return formatted

class Title(object):
    def __init__(self):
        self.title_number=''
        self.duration=datetime.timedelta(hours=0)
        self.audio_tracks=[]
        self.subtitles=[]
    def __str__(self):
        formatted="Title number "+self.title_number+":" + '\n'+\
        "    Duration: "+str(self.duration)+'\n'
        for i in self.audio_tracks:
            formatted=formatted+str(i)
        for i in self.subtitles:
            formatted=formatted+str(i)
        return formatted

class AudioTrack(object):
    def __init__(self):
        self.track_number=''
        self.language=''
        self.language_code=''
        self.codec=''
        self.channels=''
        self.sample_rate=''
        self.bitrate=''
    def __str__(self):
        formatted="Audio Track number "+self.track_number+":" +'\n'+\
        "    Language: "+self.language +'\n'+\
        "    Language Code: "+self.language_code +'\n'+\
        "    Codec: "+self.codec +'\n'+\
        "    Channels: "+self.channels +'\n'+\
        "    Sample Rate: "+self.sample_rate+"Hz" +'\n'+\
        "    Bitrate: "+self.bitrate+"bps" +'\n'
        return formatted

class Subtitle(object):
    def __init__(self):
        self.track_number=''
        self.language=''
        self.language_code=''
        self.type=''

    def __str__(self):
        formatted="Subtitle track "+self.track_number+":" +'\n'+\
        "    Language: "+self.language +'\n'+\
        "    Language_code: "+self.language_code +'\n'+\
        "    Type: " + self.type +'\n'
        return formatted

def isAudioTrack(line):
    if re.match('\+ ([0-9][0-9]*), ([A-Za-z0-9,_]*) \(([a-zA-Z0-9_ ]*)\) \(([a-zA-Z0-9_. ]*)\)',line) is None:
        return False
    else:
        return True

def parseAudioTrack(current_line):
    current_track=AudioTrack()
    trackinfo=re.match('\+ ([0-9][0-9]*), ([A-Za-z0-9,_]*) \(([a-zA-Z0-9_ ]*)\) \(([a-zA-Z0-9_. ]*)\)',current_line)
    iso639_2=re.search('iso639-2: ([a-zA-Z]*)\)',current_line)
    samples=re.search('([0-9]*)Hz',current_line)
    bitrate=re.search('([0-9]*)bps',current_line)

    current_track.track_number=trackinfo.group(1)
    current_track.language=trackinfo.group(2)
    current_track.codec=trackinfo.group(3)
    current_track.channels=trackinfo.group(4)
    if iso639_2 is not None:
        current_track.language_code=iso639_2.group(1)
    if samples is not None:
        current_track.sample_rate=samples.group(1)
    if bitrate is not None:
        current_track.bitrate=bitrate.group(1)

    return current_track

def isSubtitleTrack(current_line):
    if re.match('\+ ([0-9][0-9]*), ([A-Za-z0-9,]*) \([a-zA-Z0-9\-: ]*\) \(([a-zA-Z]*)\)\(([a-zA-Z0-9\-]*)\)',current_line) is None:
        return False
    else:
        return True

def parseSubtitleTrack(current_line):
    current_subtitle=Subtitle()
    m=re.match('\+ ([0-9][0-9]*), ([A-Za-z0-9, ]*) \(.*\) \(([a-zA-Z]*)\)\(([a-zA-Z0-9\-]*)\)',current_line)
    iso639_2=re.search('\(iso639-2: ([a-zA-Z]*)\)',current_line)

    current_subtitle.track_number=m.group(1)
    current_subtitle.language=m.group(2)
    current_subtitle.type=m.group(4)
    if iso639_2 is not None:
        current_subtitle.language_code=iso639_2.group(1)

    return current_subtitle

def parseDVD(filename):
    current_dvd=DVD()
    current_dvd.name=filename

    p = subprocess.Popen( ['HandBrakeCLI','-i',filename,'-t','0'], stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
    (output,_)=p.communicate()

    output_file=io.StringIO(output)
    output_file.seek(0)
    current_line=output_file.readline().strip()

    while current_line.startswith('+')==False and current_line!='HandBrake has exited.':        #Advance through output until we reach summary section
        m = re.match('libdvdnav: DVD Title: (.*)',current_line)
        if m != None:
            current_dvd.title=m.group(1)
        current_line=output_file.readline().strip()

    if current_dvd.title=='':
        (_,lone_filename)=os.path.split(filename)
        (current_dvd.title,_)=os.path.splitext(lone_filename)

    while current_line!='HandBrake has exited.':
        current_title=Title()
        #Have a line in the format '+ title #:'
        m = re.match('\+ title ([0-9][0-9]*)',current_line)
        if m is None:
            current_line=output_file.readline().strip()
            continue
        else:
            current_title.title_number=m.group(1)

        current_line=output_file.readline().strip()
        while re.match('\+ duration: ([0-9][0-9]):([0-9][0-9]):([0-9][0-9])',current_line) is None:
            current_line=output_file.readline().strip()
        m = re.match('\+ duration: ([0-9][0-9]):([0-9][0-9]):([0-9][0-9])',current_line)
        current_title.duration=datetime.timedelta(hours=int(m.group(1)),minutes=int(m.group(2)),seconds=int(m.group(3)))

        current_line=output_file.readline().strip() #Skip forward to audio tracks section
        while re.match('\+ audio tracks:',current_line) is None:
            current_line=output_file.readline().strip()
        current_line=output_file.readline().strip() #Defines the first audio track, presumably

        while isAudioTrack(current_line) is True:
            current_track=parseAudioTrack(current_line)
            current_title.audio_tracks.append(current_track)
            current_line=output_file.readline().strip()

        #When we get here we have gone through all this title's audio tracks and current_line
        #will have the value '+ subtitle tracks:'
        current_line=output_file.readline().strip() #Read either the first subtitle or the next title

        while isSubtitleTrack(current_line) is True:
            current_subtitle=parseSubtitleTrack(current_line)
            current_title.subtitles.append(current_subtitle)
            current_line=output_file.readline().strip()

        #When we get to here we have processed all subtitle tracks.  This title is done.
        #current_line is either '+ title #:' or 'HandBrake has exited.'
        current_dvd.titles.append(current_title)

    return current_dvd

if __name__ == '__main__':
    g=parseDVD('/mnt/cluster-programs/handbrake/jobs/tmp')
    print(g)


from pyspark.sql import SparkSession, HiveContext
from pyspark.sql.functions import lit
from pyspark.sql import Row

spark = SparkSession.builder.appName("union").master("local[*]").getOrCreate()

def customUnion(df1, df2):
    cols1 = df1.columns
    cols2 = df2.columns
    total_cols = sorted(cols1 + list(set(cols2) - set(cols1)))
    def expr(mycols, allcols):
        def processCols(colname):
            if colname in mycols:
                return colname
            else:
                return lit(None).alias(colname)
        cols = map(processCols, allcols)
        return list(cols)
    appended = df1.select(expr(cols1, total_cols)).union(df2.select(expr(cols2, total_cols)))
    return appended

data = [
    Row(zip_code=58542, dma='MIN'),
    Row(zip_code=58701, dma='MIN'),
    Row(zip_code=57632, dma='MIN'),
    Row(zip_code=58734, dma='MIN')
]

firstDF = spark.createDataFrame(data)

data = [
    Row(zip_code='534', name='MIN'),
    Row(zip_code='353', name='MIN'),
    Row(zip_code='134', name='MIN'),
    Row(zip_code='245', name='MIN')
]

secondDF = spark.createDataFrame(data)

customUnion(firstDF,secondDF).show()


import shutil
import tempfile
from django.contrib.auth import get_user_model
from django.test import Client, TestCase, override_settings
from django.urls import reverse
from django.conf import settings
from django.core.files.uploadedfile import SimpleUploadedFile
from django.core.cache import cache

from posts.models import Group, Post, Follow
from posts.forms import PostForm, Comment

User = get_user_model()

TEMP_MEDIA_ROOT = tempfile.mkdtemp(dir=settings.BASE_DIR)

@override_settings(MEDIA_ROOT=TEMP_MEDIA_ROOT)
class PostsViewsTest(TestCase):
    @classmethod
    def setUpClass(cls):
        super().setUpClass()
        small_gif = (
            b'\x47\x49\x46\x38\x39\x61\x02\x00'
            b'\x01\x00\x80\x00\x00\x00\x00\x00'
            b'\xFF\xFF\xFF\x21\xF9\x04\x00\x00'
            b'\x00\x00\x00\x2C\x00\x00\x00\x00'
            b'\x02\x00\x01\x00\x00\x02\x02\x0C'
            b'\x0A\x00\x3B'
        )
        uploaded = SimpleUploadedFile(
            name='small.gif',
            content=small_gif,
            content_type='image/gif'
        )
        cls.user = User.objects.create_user(username='TestUser')
        cls.another_user = User.objects.create_user(username='tiger')
        cls.group = Group.objects.create(
            title='Тестовая группа',
            slug='test_slug',
            description='Тестовое описание',
        )

        cls.post = Post.objects.create(
            author=cls.user,
            text='Тестовый текст',
            group=cls.group,
            image=uploaded
        )
        cls.context_test = {
            reverse('posts:index'): 'Главная страница',
            reverse(
                'posts:group_list',
                kwargs={'slug': cls.group.slug}
            ): 'Страница групп',
            reverse(
                'posts:profile',
                kwargs={'username': cls.post.author}
            ): 'Страница профиля',
            reverse(
                'posts:post_detail',
                kwargs={'post_id': cls.post.id}
            ): 'Страница деталей поста',
            reverse(
                'posts:post_edit',
                kwargs={'post_id': cls.post.id}
            ): 'Страница создания поста',
        }
        cls.post_with_views = {
            reverse('posts:index'): 'Главная страница',
            reverse(
                'posts:group_list',
                kwargs={'slug': cls.group.slug}
            ): 'Страница групп',
            reverse(
                'posts:profile',
                kwargs={'username': cls.post.author}
            ): 'Страница профиля',
        }
        cls.templates_pages_names = {
            'posts/index.html': reverse('posts:index'),
            'posts/group_list.html': reverse(
                'posts:group_list',
                kwargs={'slug': cls.group.slug}
            ),
            'posts/profile.html': reverse(
                'posts:profile',
                kwargs={'username': cls.post.author}
            ),
            'posts/post_detail.html': reverse(
                'posts:post_detail',
                kwargs={'post_id': cls.post.id}
            ),
            'posts/create_post.html': reverse(
                'posts:post_edit',
                kwargs={'post_id': cls.post.id}
            ),
            'posts/follow.html': reverse(
                'posts:follow_index'
            )
        }
        cls.form = PostForm()

    @classmethod
    def tearDownClass(cls):
        super().tearDownClass()
        shutil.rmtree(TEMP_MEDIA_ROOT, ignore_errors=True)

    def setUp(self):
        self.quest_client = Client()
        self.authorized_client = Client()
        self.authorized_client.force_login(PostsViewsTest.post.author)
        self.another_authorized_client = Client()
        self.another_authorized_client.force_login(self.another_user)

    def test_pages_uses_correct_template(self):
        """URL-адрес использует соответствующий шаблон."""
        for template, reverse_name in self.templates_pages_names.items():
            with self.subTest(reverse_name=reverse_name):
                response = self.authorized_client.get(reverse_name)
                self.assertTemplateUsed(response, template)

    def test_pages_uses_correct_template_for_guest(self):
        """URL-адрес не использует шаблон для
         не авторизованного пользователя"""
        response = self.quest_client.get(
            reverse(
                'posts:post_edit',
                kwargs={'post_id': self.post.id}
            )
        )
        self.assertTemplateNotUsed(response, 'posts/create_post.html')

    def test_all_views_page_show_correct_context(self):
        """Шаблоны views сформированы с правильным контекстом."""
        cache.clear()
        for address, temp in self.context_test.items():
            with self.subTest(temp=temp):
                danger_message = f'Шаблон {temp} сформирован не верно'
                response = (self.authorized_client.get(address))
                post = response.context['post']
                self.assertEqual(post.author, self.post.author, danger_message)
                self.assertEqual(post.text, self.post.text, danger_message)
                self.assertEqual(post.group, self.post.group, danger_message)
                self.assertEqual(post.image, self.post.image, danger_message)

    def test_post_with_group_index_show_correct_context(self):
        """Пост с группой есть на главной странице."""
        post_2 = Post.objects.create(
            author=self.user,
            text='Тестовый точно текст?',
            group=self.group,
            pub_date='2022-08-14 00:00:00',
        )
        for address, temp in self.post_with_views.items():
            with self.subTest(temp=temp):
                danger_message = f'Пост {temp} не найден на странице'
                response = (self.authorized_client.get(address))
                post = response.context['page_obj'][0]
                self.assertEqual(post.pk, post_2.pk, danger_message)

    def test_authorized_client_can_comment(self):
        """Авторизованный пользователь может комментировать посты."""
        comments_initial = Comment.objects.count()
        self.authorized_client.post(
            reverse(
                'posts:add_comment',
                kwargs={'post_id': self.post.id}
            ),
            data={'text': 'Комментарий для поста 2'}
        )
        comments_finally = Comment.objects.count()
        comments_text = Comment.objects.all()[0].text
        self.assertEqual(comments_initial + 1, comments_finally)
        self.assertEqual('Комментарий для поста 2', comments_text)

    def test_guest_client_cant_comment(self):
        """Неавторизованный пользователь не может комментировать посты."""
        comments_initial = Comment.objects.count()
        self.quest_client.post(
            reverse(
                'posts:add_comment',
                kwargs={'post_id': self.post.id}
            ),
            data={'text': 'Комментарий для поста 1'}
        )
        comments_finally = Comment.objects.count()
        self.assertNotEqual(comments_initial + 1, comments_finally)

    def test_comment_in_page_detail(self):
        """После успешной отправки комментарий появляется на странице поста."""
        self.authorized_client.post(
            reverse(
                'posts:add_comment',
                kwargs={'post_id': self.post.id}
            ),
            data={'text': 'Комментарий для поста 2'}
        )
        response = self.authorized_client.get(
            reverse(
                'posts:post_detail',
                kwargs={'post_id': self.post.id}
            )
        )
        comments_text = response.context['comments'][0].text
        self.assertEqual('Комментарий для поста 2', comments_text)

    def test_cache_home_page(self):
        """Проверка работы кеша главной страницы."""
        cache.clear()
        post_del_cache = Post.objects.create(
            text='Текст для проверки',
            author=self.user,
        )

        def response():
            return self.quest_client.get(reverse('posts:index'))

        response_primary = response().content
        post_del_cache.delete()
        response_secondary = response().content
        self.assertEqual(response_primary, response_secondary)
        cache.clear()
        response_cache_clear = response().content
        self.assertNotEqual(response_secondary, response_cache_clear)

    def test_create_follow_authorized(self):
        """Тест создания подписок авторизованным пользователем."""
        self.authorized_client.get(
            reverse('posts:profile_follow',
                    kwargs={'username': self.another_user.username}
                    )
        )
        self.assertTrue(Follow.objects.filter(author=self.another_user,
                                              user=self.user))

    def test_create_follow_non_authorized(self):
        """Тест создания подписок неавторизованным пользователем."""
        self.quest_client.get(
            reverse('posts:profile_follow',
                    kwargs={'username': self.another_user.username}
                    )
        )
        self.assertFalse(Follow.objects.filter(author=self.another_user,
                                               user=self.user))

    def test_create_follow_author(self):
        """Тест автор подписки не может подписаться сам на себя."""
        self.authorized_client.get(
            reverse('posts:profile_follow',
                    kwargs={'username': self.user.username}
                    )
        )
        self.assertFalse(Follow.objects.filter(author=self.user,
                                               user=self.user))

    def test_delete_follow_authorized(self):
        """Тест удаления подписок авторизованным пользователем."""
        self.authorized_client.get(
            reverse('posts:profile_unfollow',
                    kwargs={'username': self.another_user.username}
                    )
        )
        self.assertFalse(Follow.objects.filter(author=self.another_user,
                                               user=self.user))

    def test_new_post(self):
        """Тест новой записи пользователя появляется в ленте подписавшихся."""
        me = User.objects.create(username='me')
        self.authorized_client.get(
            reverse('posts:profile_follow',
                    kwargs={'username': me.username}
                    )
        )
        new_post = Post.objects.create(author=me, text='Test 1')
        response = self.authorized_client.get(
            reverse('posts:follow_index')
        )
        self.assertEqual(new_post, response.context['page_obj'][0])
        response = self.another_authorized_client.get(
            reverse('posts:follow_index')
        )
        self.assertEqual(len(response.context['page_obj']), 0)


import os
import pathlib
from typing import List

import pytest
from unittestadapter.discovery import (
    DEFAULT_PORT,
    discover_tests,
    parse_discovery_cli_args,
)
from unittestadapter.utils import TestNodeTypeEnum, parse_unittest_args

from .helpers import TEST_DATA_PATH, is_same_tree

@pytest.mark.parametrize(
    "args, expected",
    [
        (["--port", "6767", "--uuid", "some-uuid"], (6767, "some-uuid")),
        (["--foo", "something", "--bar", "another"], (int(DEFAULT_PORT), None)),
        (["--port", "4444", "--foo", "something", "--port", "9999"], (9999, None)),
        (
            ["--uuid", "first-uuid", "--bar", "other", "--uuid", "second-uuid"],
            (int(DEFAULT_PORT), "second-uuid"),
        ),
    ],
)
def test_parse_cli_args(args: List[str], expected: List[str]) -> None:
    """The parse_cli_args function should parse and return the port and uuid passed as command-line options.

    If there were no --port or --uuid command-line option, it should return default values).
    If there are multiple options, the last one wins.
    """
    actual = parse_discovery_cli_args(args)

    assert expected == actual

@pytest.mark.parametrize(
    "args, expected",
    [
        (
            ["-s", "something", "-p", "other*", "-t", "else"],
            ("something", "other*", "else"),
        ),
        (
            [
                "--start-directory",
                "foo",
                "--pattern",
                "bar*",
                "--top-level-directory",
                "baz",
            ],
            ("foo", "bar*", "baz"),
        ),
        (
            ["--foo", "something"],
            (".", "test*.py", None),
        ),
    ],
)
def test_parse_unittest_args(args: List[str], expected: List[str]) -> None:
    """The parse_unittest_args function should return values for the start_dir, pattern, and top_level_dir arguments
    when passed as command-line options, and ignore unrecognized arguments.
    """
    actual = parse_unittest_args(args)

    assert actual == expected

def test_simple_discovery() -> None:
    """The discover_tests function should return a dictionary with a "success" status, a uuid, no errors, and a test tree
    if unittest discovery was performed successfully.
    """
    start_dir = os.fsdecode(TEST_DATA_PATH)
    pattern = "discovery_simple*"
    file_path = os.fsdecode(pathlib.PurePath(TEST_DATA_PATH / "discovery_simple.py"))

    expected = {
        "path": start_dir,
        "type_": TestNodeTypeEnum.folder,
        "name": ".data",
        "children": [
            {
                "name": "discovery_simple.py",
                "type_": TestNodeTypeEnum.file,
                "path": file_path,
                "children": [
                    {
                        "name": "DiscoverySimple",
                        "path": file_path,
                        "type_": TestNodeTypeEnum.class_,
                        "children": [
                            {
                                "name": "test_one",
                                "path": file_path,
                                "type_": TestNodeTypeEnum.test,
                                "lineno": "14",
                                "id_": file_path
                                + "\\"
                                + "DiscoverySimple"
                                + "\\"
                                + "test_one",
                            },
                            {
                                "name": "test_two",
                                "path": file_path,
                                "type_": TestNodeTypeEnum.test,
                                "lineno": "17",
                                "id_": file_path
                                + "\\"
                                + "DiscoverySimple"
                                + "\\"
                                + "test_two",
                            },
                        ],
                        "id_": file_path + "\\" + "DiscoverySimple",
                    }
                ],
                "id_": file_path,
            }
        ],
        "id_": start_dir,
    }

    uuid = "some-uuid"
    actual = discover_tests(start_dir, pattern, None, uuid)

    assert actual["status"] == "success"
    assert is_same_tree(actual.get("tests"), expected)
    assert "errors" not in actual

def test_empty_discovery() -> None:
    """The discover_tests function should return a dictionary with a "success" status, a uuid, no errors, and no test tree
    if unittest discovery was performed successfully but no tests were found.
    """
    start_dir = os.fsdecode(TEST_DATA_PATH)
    pattern = "discovery_empty*"

    uuid = "some-uuid"
    actual = discover_tests(start_dir, pattern, None, uuid)

    assert actual["status"] == "success"
    assert "tests" not in actual
    assert "errors" not in actual

def test_error_discovery() -> None:
    """The discover_tests function should return a dictionary with an "error" status, a uuid, the discovered tests, and a list of errors
    if unittest discovery failed at some point.
    """
    # Discover tests in .data/discovery_error/.
    start_path = pathlib.PurePath(TEST_DATA_PATH / "discovery_error")
    start_dir = os.fsdecode(start_path)
    pattern = "file*"

    file_path = os.fsdecode(start_path / "file_two.py")

    expected = {
        "path": start_dir,
        "type_": TestNodeTypeEnum.folder,
        "name": "discovery_error",
        "children": [
            {
                "name": "file_two.py",
                "type_": TestNodeTypeEnum.file,
                "path": file_path,
                "children": [
                    {
                        "name": "DiscoveryErrorTwo",
                        "path": file_path,
                        "type_": TestNodeTypeEnum.class_,
                        "children": [
                            {
                                "name": "test_one",
                                "path": file_path,
                                "type_": TestNodeTypeEnum.test,
                                "lineno": "14",
                                "id_": file_path
                                + "\\"
                                + "DiscoveryErrorTwo"
                                + "\\"
                                + "test_one",
                            },
                            {
                                "name": "test_two",
                                "path": file_path,
                                "type_": TestNodeTypeEnum.test,
                                "lineno": "17",
                                "id_": file_path
                                + "\\"
                                + "DiscoveryErrorTwo"
                                + "\\"
                                + "test_two",
                            },
                        ],
                        "id_": file_path + "\\" + "DiscoveryErrorTwo",
                    }
                ],
                "id_": file_path,
            }
        ],
        "id_": start_dir,
    }

    uuid = "some-uuid"
    actual = discover_tests(start_dir, pattern, None, uuid)

    assert actual["status"] == "error"
    assert is_same_tree(expected, actual.get("tests"))
    assert len(actual.get("errors", [])) == 1


import os
import json
import math
from flask import (
    Flask, session, Blueprint, flash, g, redirect, render_template, request, url_for
)
from flask_socketio import SocketIO, send, join_room, leave_room, emit

from werkzeug.exceptions import abort

# create and configure the app
app = Flask(__name__, instance_relative_config=True)
app.config['SECRET_KEY'] = 'secret'
socketio = SocketIO(app)
#socketio = SocketIO(app, async_mode="eventlet")

# ensure the instance folder exists
try:
    os.makedirs(app.instance_path)
except OSError:
    pass

#Contains list of existing rooms and their respective users
# 'room' contains
room_list = {}

@app.route('/', methods=('GET', 'POST'))
def index():
    if request.method == 'POST':
        room_name = request.form['room_name']
        #Make Room
        if request.form['action'] == 'make':
            if room_name not in room_list:
                room_list[room_name] = { "Users": [], "Curr_Vid": "", "Queue": [], "Votes": 0, "Curr_Time": 0}
                session['room'] = room_name
                return redirect(url_for('room', room=session.get('room')))
            else:
                print('Room already exists')
                return render_template('index.html')
        #Join Room
        if request.form['action'] == 'join':
            if room_name in room_list:
                session['room'] = room_name
                return redirect(url_for('room', room=session.get('room')))
            else:
                print('Room does not exist')
                return render_template('index.html')
    else:
        room_name = session.get('room', '')
        return render_template('index.html')

@app.route('/<string:room>')
def room(room):
    if(room in room_list):
        session['room'] = room
        return render_template('room.html')
    else:
        return redirect(url_for('index'))

@socketio.on('join')
def joined():
    session['id'] = request.sid
    room = session.get('room')
    join_room(room)
    print('User has entered : ' + room)
    print(room_list[room])
    emit('user init', room_list[room], room=room)

    @socketio.on('disconnect')
    def leave():
        disconnect()

@socketio.on('leave')
def left():
    disconnect()

@socketio.on('text')
def text(message):
    room = session.get('room')
    print('User has entered '+message+ ' in ' +session.get('room'))
    emit('message', {'message':message, 'username':session.get('username')}, room=room)

@socketio.on('add user')
def adduser(username):
    room = session.get('room')
    session['username'] = username
    room_list[room]["Users"].append(username)
    print(session.get('username') + ' has been added to room ' + room)
    print(room_list[room])
    emit('user added', username, room=room)
    emit('update users', room_list[room]["Users"], room=room)
    emit('load playlist', room_list[room]["Queue"], room=room)

@socketio.on('update curr time')
def updateCurrTime(time):
    room = session.get('room')
    room_list[room]["Curr_Time"] = time
    print(room_list[room])
    emit('set curr time', time, room=room)

@socketio.on('video paused')
def pausevideo():
    room = session.get('room')
    print('video has been paused')
    emit('pause video', room=room)

@socketio.on('video played')
def playvideo():
    room = session.get('room')
    print('video playing')
    emit('play video', room=room)

@socketio.on('video buffering')
def buffervideo(time):
    room = session.get('room')
    print('video buffering')
    emit('buffer video', time, room=room)

@socketio.on('queue video')
def queuevideo(url):
    room = session.get('room')
    room_list[room]["Queue"].append(url)
    print(room_list[room])
    emit('add to queue', room_list[room]["Queue"], room=room)

@socketio.on('nextvideo')
def nextvideo():
    room = session.get('room')
    if len(room_list[room]["Queue"]) > 0:
        next_vid = room_list[room]["Queue"][0]
        del room_list[room]["Queue"][0]
        room_list[room]["Votes"] = 0
        print(room_list[room])
        emit('load next video', next_vid, room=room)
        emit('update curr vid client', next_vid, room=room)
        emit('update queue', room_list[room]["Queue"], room=room)
        emit('update votes', room_list[room]["Votes"], room=room)

@socketio.on('update curr vid')
def updateCurrVid(url):
    room = session.get('room')
    room_list[room]["Curr_Vid"] = url
    print(room_list[room])
    emit('update curr vid client', url, room=room)

@socketio.on('load video')
def loadvideo(url):
    room = session.get('room')
    emit('load new video', url, room=room)

@socketio.on('vote')
def vote():
    room = session.get('room')
    room_list[room]["Votes"] += 1
    votes = room_list[room]["Votes"]
    numusers = len(room_list[room]["Users"])
    if votes == math.ceil(numusers/2):
        if len(room_list[room]["Queue"]) > 0:
            next_vid = room_list[room]["Queue"][0]
            del room_list[room]["Queue"][0]
            room_list[room]["Votes"] = 0
            room_list[room]["Curr_Vid"] = next_vid
            print(room_list[room])
            emit('load next video', next_vid, room=room)
            emit('update curr vid client', next_vid, room=room)
            emit('update queue', room_list[room]["Queue"], room=room)
    emit('update votes', room_list[room]["Votes"], room=room)

def disconnect():
    room = session.get('room')
    if session.get('username') is not None:
        room_list[room]["Users"].remove(session.get('username'))
        print('User has left')
        print(room_list[room])
        leave_room(room)
        emit('user left', session.get('username'), room=room)
        emit('update users', room_list[room]["Users"], room=room)
        if len(room_list[room]["Users"]) <= 0:
            del room_list[room]
            print(room_list)

if __name__ == "__main__":
    #app.run(host='0.0.0.0')
    socketio.run(app)


# flask packages
from flask import (
    Blueprint, flash, g, redirect, render_template, request, url_for, session
)
from werkzeug.exceptions import abort
from flaskr.auth import login_required
from flaskr.db import get_db
# json file extraction packages
import urllib, json
import urllib.request, json
# blockchain packages
from web3 import Web3, HTTPProvider

from datetime import datetime, date, timedelta
import calendar
# timezone package needed for prediction
from pytz import timezone
bp = Blueprint('satellite', __name__)

# N2yo API key
api_key = 'KQ9N7M-XWZL5A-NV4CAK-4NG8'
# Skyfield API prediction API
from skyfield.api import load, EarthSatellite, wgs84
#   Smart Contract data set up

# source: https://dev.to/gcrsaldanha/deploy-a-smart-contract-on-ethereum-with-python-truffle-and-web3py-5on
# blockchain smart contract code, possible to do it on this

# ganache address (EVM local blockchain network)
blockchain_address = 'http://127.0.0.1:7545'
# Client instance to interact with the blockchain
web3 = Web3(Web3.HTTPProvider(blockchain_address))
# Set the default account (so we don't need to set the "from" for every transaction call)
web3.eth.defaultAccount = web3.eth.accounts[0]
# Path to the compiled contract Satellites JSON file
compiled_contract_path = '/Users/jordanlloyd/Satellite/Code/flaskr/artifacts/Satellites.json'
# Deployed contract address (changes if I migrate --reset the contract)
deployed_contract_address = '0x26aDB61B71Dfe8eE3d9AA1Dd1eE6Ba0BE6bB9238'

# retrieves the smart contract data
with open(compiled_contract_path) as file:
    contract_json = json.load(file)  # load contract info as JSON
    contract_abi = contract_json['abi']  # fetch contract's abi - necessary to call its functions

# Fetch deployed contract reference
contract = web3.eth.contract(address=deployed_contract_address, abi=contract_abi)

# satellite home page route
@bp.route('/satelliteindex', methods=('GET', 'POST'))
def index():
    # near satellite retrieval
    nearsats = get_near()
    if nearsats is None:
        nearsats = ['NO SATELLITES', 'NEAR']
    # arrays of popular satellite details
    popnames = ['Space Station', 'SES 1', 'NOAA 19', 'GOES 13', 'TERRA', 'AQUA']
    popids = [25544, 36516, 25338, 29155, 25994, 27424]

    if request.method == 'POST':
        # NORAD ID track verification
        satelliteid = request.form['satelliteid']
        error = None
        if not satelliteid:
            error = 'Satellite ID is required.'
            flash(error)

        elif (check_sat(get_info(satelliteid)) == False):
            error = 'Satellite ID is invalid.'
            flash(error)
        elif (check_sat(get_info(satelliteid)) == True):
            session.clear()
            # redirects to the satellite tracking page
            return redirect(url_for('satellite.track', satelliteid=satelliteid))

    return render_template('satellites/index.html', nearsats=nearsats, popnames=popnames, popids=popids)

# about page template route
@bp.route('/about')
def about():
    return render_template('satellites/About.html')

# track page route with get and post methods
@bp.route('/track', methods=('GET', 'POST'))
def track():
    # retrieves the satid variable passed from the home page
    satid = request.args['satelliteid']
    # retrieves the satellite data for the satid variable
    satdata = get_info(satid)
    # presents the current time in the current time zone instead of the timestamp
    utc = getUTC(satdata['positions'][0]['timestamp'])

    if request.method == 'POST':
        satid = request.args['satelliteid']
        satdata = get_info(satid)
        # if the save button for the blockchain is selected; saves the satdata to the blockchain network

        if request.form['submit'] == 'save':

            #variables of the satellite data
            satname = satdata['info']['satname']
            satlon = satdata['positions'][0]['satlongitude']
            satlat = satdata['positions'][0]['satlatitude']
            satelev = satdata['positions'][0]['elevation']
            sataz = satdata['positions'][0]['azimuth']
            satdate = satdata['positions'][0]['timestamp']

            # save the data to the smart contract
            tx_hash = saveSat(satdata)
            # retrieves the latest/ current block number
            blockNo = web3.eth.blockNumber
            satComment = request.form['satComment']
            cdate = datetime.now()

            # save the block number and comment into the database
            # we can access the satdata using the block number
            db = get_db()
            db.execute(
                'INSERT INTO block (tx_hash, block_no, date, comment, satname, satid, longitude, latitude, elevation, azimuth, timestamp)'
                ' VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)',
                (tx_hash, blockNo, cdate.strftime("%d/%m/%Y %H:%M:%S"), satComment, satname, satid, satlon, satlat, satelev, sataz, getUTC(satdate))
            )
            db.commit()
            # redirects to the blockchain page
            return redirect(url_for('satellite.blockchain', satdata=satdata ))

        # if the predict button is selected; it will request the time frame and calculate the longitude and latitude
        elif request.form['submit'] == 'predict':
            satid = request.args['satelliteid']
            # retrieves the satellites TLE (two line elements)
            tle = get_tle(satid)

            # stores the timeframe selected
            period = request.form['predict']

            # retrieves the longitude and latitude of the current tle
            lon, lat = pred_longlat(tle, period)

            satelliteid = satid

            return render_template('satellites/Track.html',lon=lon, lat=lat, utc=utc, satelliteid=satelliteid, satdata=satdata )

    #  redirected from the home page, displays the desired satellites data and the location on the map
    if request.method == 'GET':
        satelliteid = request.args['satelliteid']

        satdata = get_info(satelliteid)

        return render_template('satellites/Track.html', satdata=satdata, utc=utc, satelliteid=satelliteid)

# blockchain page route; retrieves the blockchain database
@bp.route('/blocks')
def blockchain():
    db = get_db()
    blocks = db.execute(
        'SELECT tx_hash, block_no, date, comment, satname, satid, longitude, latitude, elevation, azimuth, timestamp'
        ' FROM block'
    ).fetchall()
    # passes the database information to the html page
    return render_template('satellites/viewBlocks.html', blocks=blocks)

# API requires the long and latitude of the user;
# To avoid data security concerns; I will assume the user is using the website from manchester
# Manchester coordinates
#longitude = -2.23743
#latitude = 53.48095

# function retrieves the satellite data to a python dictionary
def get_info(satelliteid):
    #formats the satellite id and the api key into the link
    positionurl = "https://api.n2yo.com/rest/v1/satellite/positions/{}/53.48095/-2.23743/38/1/&apiKey={}".format(satelliteid, api_key)

    # json data parsed into python dictionary
    with urllib.request.urlopen(positionurl) as responsepos:
        data = json.loads(responsepos.read())

    return data

# function verifies the existance of the satellite
def check_sat(satdata):
    #get value of a key and checks the satname attribute has a value
    # the url always returns a json
    if satdata['info']['satname'] == None:
        return False
    else:
        return True

# function changes the variable types of the satellite data to type string
def satToStr(testsat):
    satarray = [str(testsat['info']['satname']), str(testsat['positions'][0]['satlongitude']),
    str(testsat['positions'][0]['satlatitude']), str(testsat['positions'][0]['elevation']) , str(testsat['positions'][0]['azimuth']), str(testsat['positions'][0]['timestamp'])]
    return satarray

# corrects the time to UTC form, use when displaying result
def getUTC(timestamp: int):
    utc = datetime.utcfromtimestamp(timestamp).strftime('%Y-%m-%d %H:%M:%S')
    return utc

# saves the current satellite info to the smart contract state memory, thus creating a new block
def saveSat(testsat):
    #sets the satellite data and creates the hash of the save
    tx_hash = contract.functions.set(0 ,testsat['info']['satid'], str(testsat['info']['satname']), str(testsat['positions'][0]['satlongitude']), str(testsat['positions'][0]['satlatitude']),
    str(testsat['positions'][0]['elevation']), str(testsat['positions'][0]['azimuth']), str(testsat['positions'][0]['timestamp'])).transact()
    #provides a waiting time for the transacation to be complete and mined
    # i.e stops executing any code until the block's been developed
    web3.eth.waitForTransactionReceipt(tx_hash)
    return tx_hash

# calls the smart contract get function to read the current blocks variable states, returns an array
def callSat():
    satinfo = contract.functions.get().call()
    return satinfo

# function retrieves the tle (two line elements) of a satellite
def get_tle(satid):
    #formats the tle into a json
    tlejson = "https://api.n2yo.com/rest/v1/satellite/tle/25544&apiKey={}".format(api_key)
    with urllib.request.urlopen(tlejson) as responsepos:
        data = json.loads(responsepos.read())
    tle = data['tle']
    return(tle)

def pred_longlat(tle, period):
    #time abstraction
    ts = load.timescale()
    # splits the TLE elements into their lines
    line1, line2 = tle.splitlines()
    # represents the tle data as the satellite
    satellite = EarthSatellite(line1, line2)
    #utc(year, month=1, day=1, hour=0, minute=0, second=0.0)
    #time = ts.utc(year, month, day, hour)
    # retrieves the correct timezone
    uk = timezone('Europe/London')

    # adds time or dates to the current datetime
    if period == 1 :
        dt = datetime.now() + timedelta(hours=1)
    elif period == 2:
        dt = datetime.now() + timedelta(hours=2)
    elif period == 4:
        dt = datetime.now() + timedelta(hours=4)
    elif period == 6:
        dt = datetime.now() + timedelta(hours=6)
    elif period == 12:
        dt = datetime.now() + timedelta(hours=12)
    elif period == 24:
        dt = datetime.now() + timedelta(days=1)
    elif period == 40:
        dt = datetime.now() + timedelta(days=4)
    else: # period == 70
        dt = datetime.now() + timedelta(days=7)

    predTime = uk.localize(dt)
    time = ts.from_datetime(predTime)

    # gets the satellite data at the predicted time
    #t1 = ts.utc(2014, 1, 24)
    predloc = satellite.at(time)

    #stores the values of the satellites positions printed onto the earths surface (directly below it)
    subsat = wgs84.subpoint(predloc)
    # rounds the results to 8 decimal places for the google map location
    lon = round(subsat.longitude.degrees, 6)
    lat = round(subsat.latitude.degrees, 6)
    return(lon, lat)

# function to retrieve nearby satellite names and information to manchester within 4degrees above
def get_near():
    # json data parsed into python dictionary
    nearjson = "https://api.n2yo.com/rest/v1/satellite/above/53.48095/-2.23743/38/4/0/&apiKey={}".format(api_key)

    with urllib.request.urlopen(nearjson) as responsepos:
        data = json.loads(responsepos.read())
    # retreives the list of satellites and their data
    # validates there are some satellites, to prevent errors
    if data is None:
        return None
    else:
        nearsat = data['above']
        return nearsat

def setPredtime(period):
    day = date.today().day
    month = date.today().month
    year = date.today().year
    now = datetime.now()
    hour = now.hour
    # make shift 'switch' and validation to ensure the time is correct for prediction

    # 1hr
    if(period == 1):
        hour = hour + 1
        if (hour >= 24 ):
            day = day + 1
            hour = hour - 24
            if (day > calendar.monthrange(year, month)[1] ):
                day = day - calendar.monthrange(year, month)[1]
                month = month + 1
                if (month >= 13):
                    month = 1
                    year = year + 1
    # 2hrs
    elif (period == 2):
        hour = hour + 2
        if (hour >= 24 ):
            day = day + 1
            hour = hour - 24
            if (day > calendar.monthrange(year, month)[1] ):
                day = day - calendar.monthrange(year, month)[1]
                month = month + 1
                if (month >= 13):
                    month = 1
                    year = year + 1
    # 4hrs
    elif(period == 4):
        hour = hour + 4
        if (hour >= 24 ):
            day = day + 1
            hour = hour - 24
            if (day > calendar.monthrange(year, month)[1] ):
                day = day - calendar.monthrange(year, month)[1]
                month = month + 1
                if (month >= 13):
                    month = 1
                    year = year + 1
    # 6hrs
    elif(period == 6):
        hour = hour + 6
        if (hour >= 24 ):
            day = day + 1
            hour = hour - 24
            if (day > calendar.monthrange(year, month)[1]):
                day = day - calendar.monthrange(year, month)[1]
                month = month + 1
                if (month >= 13):
                    month = 1
                    year = year + 1
    # 12hrs
    elif(period == 12):
        hour = hour + 12
        if (hour >= 24 ):
            day = day + 1
            hour = hour - 24
            if (day > calendar.monthrange(year, month)[1] ):
                day = day - calendar.monthrange(year, month)[1]
                month = month + 1
                if (month >= 13):
                    month = 1
                    year = year + 1
    # 1day
    elif(period == 24):
        day = day + 1
        if (day > calendar.monthrange(year, month)[1] ):
            day = day - calendar.monthrange(year, month)[1]
            month = month + 1
            if (month >= 13):
                month = 1
                year = year + 1
    # 4days
    elif(period == 40):
        day = day + 4
        if (day > calendar.monthrange(year, month)[1] ):
            day = day - calendar.monthrange(year, month)[1]
            month = month + 1
            if (month >= 13):
                month = 1
                year = year + 1
    # 7days
    elif(period == 70):
        day = day + 7
        if (day > calendar.monthrange(year, month)[1] ):
            day = day - calendar.monthrange(year, month)[1]
            month = month + 1
            if (month >= 13):
                month = 1
                year = year + 1

    return (year, month, day, hour)


from blessed import Terminal

class CliInterface:
    def __init__(self) -> None:
        self.__term = Terminal()
        self.cli_list = []

    def __print_screen(self, msg, input, cursor_pos):
        line = 0
        title = "|    Python Chat - (RabbitMQ)    |"
        with self.__term.location(0, line):
            print(title)
            line += 1
        with self.__term.location(0, line):
            print("-" * len(title))
            line += 1
        for str_line in self.cli_list:
            with self.__term.location(0, line):
                print(f"{str_line:70}")
                line += 1
        with self.__term.location(0, line):
            print("-" * len(title))
            line += 1

        with self.__term.location(0, line):
            f_input = input + " "
            f_input = f_input[:cursor_pos] + self.__term.reverse(f_input[cursor_pos]) + input[(cursor_pos+1):]
            print(f"{msg}: {f_input:}   ")

    def start_input(self, mode_args:dict):
        with self.__term.fullscreen(), self.__term.hidden_cursor():
            while mode_args['mode'] != 'end':
                text = ''
                cursor_pos = 0
                while True:
                    mode_args[mode_args['mode']]['update'](self.cli_list)

                    self.__print_screen(mode_args[mode_args['mode']]['prompt'], text, cursor_pos)

                    with self.__term.cbreak():
                        key = self.__term.inkey(timeout=1)

                    if key == '\n':
                        break
                    elif key == '':
                        continue
                    elif key.isprintable():
                        text = text[:cursor_pos] + key + text[cursor_pos:]
                        cursor_pos += 1
                    elif key.code == self.__term.KEY_BACKSPACE:
                        if cursor_pos > 0:
                            text = text[:cursor_pos-1] + text[cursor_pos:]
                            cursor_pos -= 1
                    elif key.code == self.__term.KEY_DELETE:
                        if cursor_pos < len(text) and len(text) > 0:
                            text = text[:cursor_pos] + text[cursor_pos+1:]
                    elif key.code == self.__term.KEY_LEFT:
                        if cursor_pos > 0:
                            cursor_pos -= 1
                    elif key.code == self.__term.KEY_RIGHT:
                        if cursor_pos < len(text):
                            cursor_pos += 1
                mode_args[mode_args['mode']]['send'](text)
                print(self.__term.clear())


import os
import toml
from collections.abc import Mapping, Iterable
from urllib.parse import urlparse, urlunparse

setup_fields = (
    'install_requires',
    'tests_require',
    'dependency_links',
    'extras_require',
    'python_requires',
)

url_keys = {
    'path',
    'file',
    'git',
    'svn',
    'hg',
    'bzr',
    'uri',
}

marker_keys = {
    'implementation_name',
    'implementation_version',
    'os_name',
    'platform_machine',
    'platform_python_implementation',
    'platform_release',
    'platform_system',
    'platform_version',
    'python_full_version',
    'python_version',
    'sys_platform',
}

other_keys = {
    'editable',
    'extras',
    'index',
    'markers',
    'ref',
    'version',
}

def deauth_url(url):
    u = urlparse(url)
    return urlunparse(u[:1] + (u[1].rsplit('@', 1)[-1],) + u[2:])

def clone_config(cfg, interpolate=False):
    if isinstance(cfg, Mapping):
        return {k: clone_config(v, interpolate) for k, v in cfg.items()}
    elif isinstance(cfg, str):
        return os.path.expandvars(cfg) if interpolate else cfg
    elif isinstance(cfg, Iterable):
        return [clone_config(i, interpolate) for i in cfg]
    else:
        return cfg

class Dependency(dict):
    def __init__(self, name, data, sources, indexes):
        data = {'version': data} if isinstance(data, str) else data
        super().__init__(data)
        self['name'] = name
        self['sources'] = sources
        self['indexes'] = indexes

    @property
    def name(self):
        return self['name']

    @property
    def url(self):
        keys = list(url_keys & self.keys())
        if not keys and 'index' not in self:
            return ''
        if len(keys) > (1 - ('index' in self)):
            raise ValueError(
                'Pipfile Dependency[{0}] conflict: {1!r}'.format(
                    self.name, keys))
        if keys and 'version' in self:
            raise ValueError(
                'Pipfile Dependency[{0}] conflict: {1!r}'.format(
                    self.name, ['version'] + keys))
        url = keys[0] if keys else 'index'
        if url in ('git', 'svn', 'hg', 'bzr'):
            url = '{0}+{1}'.format(url, self[url])
        elif url == 'index':
            if not self['indexes']:
                return ''
            url = '{0}/{1}/'.format(self['sources'][self['index']], self.name)
        else:
            url = self[url]
        if 'ref' in self:
            url = '{0}@{1[ref]}#egg={1.name}'.format(url, self)
        return ' @ {0}'.format(url)

    @property
    def extras(self):
        if 'extras' in self:
            return '[{0}]'.format(','.join(self['extras']))
        return ''

    @property
    def specifier(self):
        if 'version' in self:
            if self['version'] != '*':
                return self['version']
        return ''

    @property
    def marker(self):
        markers = self.get('markers', '')
        keys = ['{} {}'.format(k, self[k]) for k in marker_keys & self.keys()]
        if keys and markers:
            markers = ' and '.join([' ; ({0})'.format(markers)] + keys)
        elif keys:
            markers = ' ; ' + ' and '.join(keys)
        elif markers:
            markers = ' ; ' + markers
        else:
            markers = ''
        return markers

    def __str__(self):
        return '{0.name}{0.extras}{0.specifier}{0.url}{0.marker}'.format(self)

class Pipfile(dict):

    def __init__(self, path, interpolate=False, pythons=False, extras=False, indexes=False):
        self.interpolate = interpolate
        self.indexes = indexes
        self.extras = {'table': 'extra', 'key': 'name', 'subtable': 'packages'}

        if extras is True:
            pass
        elif extras and isinstance(extras, int):
            self.extras['style'] = extras
        elif isinstance(extras, str):
            self.extras['table'] = extras
        elif isinstance(extras, Mapping):
            self.extras.update(extras)
        else:
            self.extras['table'] = None

        if pythons is True:
            self.pythons = '=={0[requires][python_version]}'
        else:
            self.pythons = pythons

        if isinstance(path, Mapping):
            data = path
            self.path = 'Pipfile'
        else:
            self.path = str(path)

            with open(self.path, 'r') as fp:
                data = toml.load(fp)

        super().__init__(clone_config(data, interpolate))

    @property
    def sources(self):
        return {
            s['name']: s['url'].rstrip('/') if not self.interpolate
            else deauth_url(s['url'].rstrip('/'))
            for s in self['source']
        }

    def get_deps(self, table):
        return [
            Dependency(name, spec, self.sources, self.indexes)
            for name, spec in table.items()
            if isinstance(spec, str) or spec.get('path') != '.'
        ]

    @property
    def deps(self):
        return self.get_deps(self.get('packages', {}))

    @property
    def dev_deps(self):
        return self.get_deps(self.get('dev-packages', {}))

    @property
    def extra_deps(self):
        if self.extras.get('style', 1) == 3:
            return {
                k[:-9]: self.get_deps(v)
                for k, v in self.items()
                if k.endswith('-packages')
            }
        if self.extras.get('style', 1) != 2:
            return {
                k: self.get_deps(v)
                for k, v in self.get(self.extras['table'], {}).items()
            }
        return {
            e[self.extras['key']]: self.get_deps(e[self.extras['subtable']])
            for e in self.get(self.extras['table'], [])
        }

    @property
    def all_deps(self):
        return self.deps + self.dev_deps + [
            d for deps in self.extra_deps.values() for d in deps
        ]

    @property
    def install_requires(self):
        return sorted(str(p) for p in self.deps)

    @property
    def tests_require(self):
        return sorted(str(d) for d in self.dev_deps)

    @property
    def dependency_links(self):
        if self.indexes is False:
            return []
        return sorted(
            '{0}/{1}/'.format(self.sources[d['index']], d.name)
            for d in self.all_deps if 'index' in d
        )

    @property
    def extras_require(self):
        if not self.extras['table']:
            return None
        return {k: sorted(str(d) for d in v) for k, v in self.extra_deps.items()}

    @property
    def python_requires(self):
        if not isinstance(self.pythons, str):
            return None
        return self.pythons.format(self)

    def setup_kwargs(self):
        return {
            k: v for k, v in (
                (key, getattr(self, key)) for key in setup_fields
            ) if v
        }


"""
汽车耗油预测实战
"""
import tensorflow as tf
import pandas as pd
from tensorflow import keras
import matplotlib
from tensorflow.keras import layers, losses

matplotlib.use("TkAgg")
import matplotlib.pyplot as plt

dataset_path = keras.utils.get_file('auto-mpg.data',"http://archive.ics.uci.edu/ml/machine-learning-databases/auto-mpg/auto-mpg.data")
col =  ['MPG','Cylinders','Displacement','Horsepower','Weight','Acceleration', 'Model Year', 'Origin']
raw_dataset = pd.read_csv(dataset_path,names=col,na_values="?",
                          comment='\t',sep=" ",skipinitialspace=True)

dataset = raw_dataset.copy()
#统计数据的特征
blank_num = dataset.isna().sum()
# print(blank_num)
#delete balnk data
dataset = dataset.dropna()
blank_num = dataset.isna().sum()
# print(blank_num)

#progress data 预处理数据
origin = dataset.pop('Origin')
dataset['USA'] = (origin == 1)*1.0
dataset['Europe'] = (origin == 2)*1.0
dataset['Japen'] = (origin == 3)*1.0
print(dataset.tail())

#切分训练集 数据集
train_dataset = dataset.sample(frac=0.8,random_state=0)
test_dataset = dataset.drop(train_dataset.index)

train_labels = train_dataset.pop("MPG")
test_labels = test_dataset.pop("MPG")

#通过标准差和均值 对数据进行规范化
train_stats = train_dataset.describe()
#转置
train_stats = train_stats.transpose()
"""
print(train_stats)
              count         mean         std  ...     50%      75%     max
Cylinders     314.0     5.477707    1.699788  ...     4.0     8.00     8.0
Displacement  314.0   195.318471  104.331589  ...   151.0   265.75   455.0
Horsepower    314.0   104.869427   38.096214  ...    94.5   128.00   225.0
Weight        314.0  2990.251592  843.898596  ...  2822.5  3608.00  5140.0
Acceleration  314.0    15.559236    2.789230  ...    15.5    17.20    24.8
Model Year    314.0    75.898089    3.675642  ...    76.0    79.00    82.0
USA           314.0     0.624204    0.485101  ...     1.0     1.00     1.0
Europe        314.0     0.178344    0.383413  ...     0.0     0.00     1.0
Japen         314.0     0.197452    0.398712  ...     0.0     0.00     1.0
"""
def formater(x):
    return (x-train_stats['mean'])/train_stats['std']

#标准化
format_train_data = formater(train_dataset)
format_test_data = formater(test_dataset)

"""
314行 输入特征长度为9
print(format_train_data.shape,train_labels.shape)
(314, 9) (314,)
"""
#构建dateset对象，并随机打乱之后分批次
train_db = tf.data.Dataset.from_tensor_slices((format_train_data.values,train_labels.values)).shuffle(100).batch(32)

#查看属性对于 MPG的影响
"""
fig,(ax1,ax2,ax3) = plt.subplots(1,3,sharey=True)
ax1.scatter(train_dataset["Cylinders"],train_labels)
ax2.scatter(train_dataset["Displacement"],train_labels)
ax3.scatter(train_dataset["Model Year"],train_labels)
plt.show()
"""

#定义网络
class Network(keras.Model):
    def __init__(self):
        super(Network, self).__init__()
        self.lay1 = layers.Dense(64,activation='relu')
        self.lay2 = layers.Dense(64,activation='relu')
        self.lay3 = layers.Dense(1)

    def call(self,inputs,training=None,mask=None):
        x = self.lay1(inputs)
        x = self.lay2(x)
        x = self.lay3(x)

        return x

model = Network()
#9 为特征值长度
model.build(input_shape=(4,9))
print(model.summary())
optimizer = tf.keras.optimizers.RMSprop(0.001)
for epoch in range(200):
    #遍历一次训练集
    for step,(x,y) in enumerate(train_db):
        with tf.GradientTape() as tape:
            out = model(x)
            #计算MSE均方差
            loss = tf.reduce_mean(losses.MSE(y,out))
            mae_loss = tf.reduce_mean(losses.MAE(y,out))
        if step % 10 == 0:
            print("eopch:",epoch,"step:",step,"loss:",float(mae_loss))
        grads = tape.gradient(loss,model.trainable_variables)
        optimizer.apply_gradients(zip(grads,model.trainable_variables))


# Importing all necessary python libraries
import pandas as pd
import networkx as nx
import numpy as np
import scipy
from scipy.sparse import csr_matrix
import pickle
import os
from sklearn import metrics
from model import predict_attribute
# ----------------------------------------

# ----------------------------------------
dataset = 'Amherst41'                           # 'playgraph' or 'UNC28' or 'American75' or 'Amherst41' or 'Auburn71' or 'Baylor93' or 'Berkeley13' or 'Bingham82' or 'Bowdoin47' or 'Brandeis99' or 'Brown11' or 'BU10'
prediction_type = 'classification'                  # 'classification' or 'regression'
model = 'RandomForest_hyper'                              # 'SVM' or 'RandomForest' or 'RandomForest_hyper'
predicting_attribute = 'gender'                     # 'student_fac' or 'gender' or 'major_index' or 'second_major' or 'dorm' or 'year' or 'high_school  ('Status' = 'student_fac')
selected_features = 'fea'                             # 'fea', 'adj', 'emb'
# Categories and their meaning in selected_features:
    # 'fea'          = Features
    # 'adj'          = Adjacency
    # 'emb'          = Embedding

rand_state_for_split = 15
embedding = 'Node2Vec'
# ----------------------------------------

# ----------------------------------------
# ----------------------------------------
def classification_metrics(Test_True_Labels, Test_Predicted_labels):
    acc = metrics.accuracy_score(Test_True_Labels, Test_Predicted_labels)
    f1_macro = metrics.f1_score(Test_True_Labels, Test_Predicted_labels, average='macro')
    precision_macro = metrics.precision_score(Test_True_Labels, Test_Predicted_labels, average='macro')
    recall_macro = metrics.recall_score(Test_True_Labels, Test_Predicted_labels, average='macro')
    f1_weighted = metrics.f1_score(Test_True_Labels, Test_Predicted_labels, average='weighted')
    adj_RI = metrics.adjusted_rand_score(Test_True_Labels, Test_Predicted_labels)

    return acc, f1_macro, precision_macro, recall_macro, f1_weighted, adj_RI
# ----------------------------------------
# ----------------------------------------

# ----------------------------------------
def constructor(dataset, predicting_attribute, embedding, selected_features, analysis):
    if analysis == 'no_filtering':
        with open('No_Filtered_nodes_edges/{}_hot_featuresDf_except_{}.pickle'.format(dataset, predicting_attribute), 'rb') as handle: featuresDf = pickle.load(handle)
        with open('No_Filtered_nodes_edges/{}_adjDf.pickle'.format(dataset), 'rb') as handle: adjDf = pickle.load(handle)
        with open('No_Filtered_nodes_edges/{}_col_{}.pickle'.format(dataset, predicting_attribute), 'rb') as handle: toPredictDf = pickle.load(handle)
        with open('pickles/generated_embeddings/{}_{}.pickle'.format(embedding, dataset), 'rb') as handle: embDf = pickle.load(handle)
        print('Done features = ', featuresDf.shape, toPredictDf.shape, adjDf.shape, embDf.shape)
    elif analysis == 'filtered_row':
        with open('Filtered_nodes_edges/{}_hot_featuresDf_except_{}.pickle'.format(dataset, predicting_attribute), 'rb') as handle: featuresDf = pickle.load(handle)
        with open('Filtered_nodes_edges/{}_adjDf.pickle'.format(dataset), 'rb') as handle: adjDf = pickle.load(handle)
        with open('Filtered_nodes_edges/{}_col_{}.pickle'.format(dataset, predicting_attribute), 'rb') as handle: toPredictDf = pickle.load(handle)
        with open('pickles/generated_embeddings/{}_{}_ones_3_only_adj.pickle'.format(dataset, embedding), 'rb') as handle: embDf = pickle.load(handle)
    elif analysis == 'filtered_row_col':
        with open('Filtered_nodes_edges/{}_hot_featuresDf_except_{}.pickle'.format(dataset, predicting_attribute), 'rb') as handle: featuresDf = pickle.load(handle)
        with open('Filtered_nodes_edges/{}_adjDf_fullGraph_filterRow.pickle'.format(dataset), 'rb') as handle: adjDf = pickle.load(handle)
        with open('Filtered_nodes_edges/{}_col_{}.pickle'.format(dataset, predicting_attribute), 'rb') as handle: toPredictDf = pickle.load(handle)
        with open('Filtered_nodes_edges/{}_embDf_fullGraph_filterRow.pickle'.format(dataset), 'rb') as handle: embDf = pickle.load(handle)
    else:
        print('wrong parameter')
        exit(0)

    if selected_features == 'fea':
        return featuresDf, toPredictDf
    elif selected_features == 'adj':
        return adjDf, toPredictDf
    elif selected_features == 'emb':
        return embDf, toPredictDf
    else:
        print('wrong prediction_type!')
        exit(0)
# ----------------------------------------

# ----------------------------------------
# Params:
#   dataset                 = a string containing dataset name
#   prediction_type         = a string ('classification' or 'regression')
#   model                   = a string
#   predicting_attribute    = a string
#   selected_features       = a string
#   rand_state_for_split    = an integer
# Return values:
#   acc, f1_macro, precision_macro, recall_macro, f1_weighted, adj_RI = a float
# def get_settings(dataset_attributes, dataset_edges, model, predicting_attribute, prediction_type, selected_features):
def do_category_specific_task_prediction(dataset, prediction_type, model, predicting_attribute, selected_features, rand_state_for_split, embedding, analysis):

    featuresDf, y = constructor(dataset, predicting_attribute, embedding, selected_features, analysis)    # Get features and labels
    print("Featurs and y collected ___________________________", featuresDf.shape, y.shape)
    # result = predict_attribute(featuresDf, y, model, prediction_type, rand_state_for_split)

    # Get labels_test and labels_predicted
    y_test, predicted_labels = predict_attribute(featuresDf, y, model, prediction_type, rand_state_for_split)

    # Get evaluation metric values
    if prediction_type == 'classification':
        acc, f1_macro, precision_macro, recall_macro, f1_weighted, adj_RI = classification_metrics(y_test.tolist(), predicted_labels)
        print("acc: ", acc, "f1_macro: ", f1_macro, "precision_macro: ", precision_macro, "recall_macro: ", recall_macro, "f1_micro: ", "f1_weighted: ", f1_weighted, "adj_RI: ", adj_RI)

    return acc, f1_macro, precision_macro, recall_macro, f1_weighted, adj_RI
# ----------------------------------------


import numpy as np
import unittest

import neurogrid as ng

class TestNonlinearCommunication(unittest.TestCase):
    def setUp(self):
        self.rng = np.random.RandomState(seed=5)

    def test_communication(self):
        N1 = 30
        N2 = 31

        D = 2
        S = 20

        nonlinear = 10
        balanced = False

        encoder_type = 'random'

        A = ng.ensemble.Ensemble(N1, N1, D, seed=1, nonlinear=nonlinear,
                                    balanced=balanced, encoder_type=encoder_type)
        B = ng.ensemble.Ensemble(N2, N2, D, seed=3, nonlinear=nonlinear,
                                    balanced=balanced, encoder_type=encoder_type)

        Ad_e, Ad_i = A.get_dual_decoder(fr_in=400, fc_in=500, fr_out=400, fc_out=500)
        Bd = B.get_decoder(fr=400, fc=500)
        Ad = A.get_decoder(fr=400, fc=500)

        XA = ng.samples.random(S, D, self.rng)

        enc = B.encoders
        #norm = np.sum(np.abs(enc), axis=1)
        #enc = enc/norm[:,None]

        ABw_e = np.dot(Ad_e, np.where(enc.T>0, enc.T, 0)) + np.dot(Ad_i, np.where(enc.T<0, -enc.T, 0))
        ABw_i = np.dot(Ad_i, np.where(enc.T>0, enc.T, 0)) + np.dot(Ad_e, np.where(enc.T<0, -enc.T, 0))

        Afs = np.zeros(N1*N1, dtype='f')
        Bfs = np.zeros(N2*N2, dtype='f')

        Ae_input, Ai_input = ng.activity.create_stimulus(XA, A.encoders, fc=500, fr=400)
        Be_input, Bi_input = ng.activity.create_stimulus(XA, B.encoders, fc=500, fr=400)

        #print Ae_input.shape, Ai_input.shape
        Arate = A.neurons.rate(Ae_input, Ai_input)

        in_e = np.dot(Arate.T, ABw_e)
        in_i = np.dot(Arate.T, ABw_i)
        #print in_e.shape, in_i.shape

        Brate = B.neurons.rate(in_e.T, in_i.T)

        output = np.dot(Brate.T, Bd)
        outputA = np.dot(Arate.T, Ad)

        import matplotlib.pyplot as plt
        plt.figure()
        plt.plot(output, linewidth=3)
        plt.plot(XA)

        plt.figure()
        plt.plot(outputA, linewidth=3)
        plt.plot(XA)

        plt.figure()
        plt.plot(output-XA)

        #plt.figure()
        #plt.scatter(Ae_input[:,0], Ai_input[:,0])

        #plt.figure()
        #plt.scatter(in_e[0,:], in_i[0,:])
        #plt.scatter(Be_input[:,0], Bi_input[:,0], color='r')

        plt.show()

        """
        print in_e.shape, in_i.shape
        import matplotlib.pyplot as plt
        plt.figure()
        plt.scatter(C.encoders[:,0], C.encoders[:,1])

        plt.figure()
        plt.scatter(Ae_input, Ai_input)
        #plt.figure()
        #plt.plot(XA, Arate.T)
        plt.figure()
        #plt.plot((in_e+in_i).T)
        #plt.scatter(in_e[:,2], in_i[:,2])
        index = 2
        plt.scatter(in_e[index,:], in_i[index,:], color='r')
        plt.scatter(in_e[index+1,:]+20, in_i[index+1,:], color='b')
        plt.scatter(in_e[index+2,:]+40, in_i[index+2,:], color='m')
        print XA[index:index+3]
        plt.show()
        1/0
        """

        """
        import matplotlib.pyplot as plt
        plt.figure()
        plt.plot(output_rate)
        plt.plot(correct)
        plt.figure()
        plt.plot(output)
        plt.plot(correct)
        plt.figure()
        plt.plot(output2)
        plt.plot(np.array(correct)/2)
        plt.show()
        """

        return

if __name__=='__main__':
    unittest.main()


"""
Due to uncertainty about how to best structure things and the size of the app,
the models and views were split into multiple individual files instead of
the traditional models.py and views.py files.

This file contains the front page view, plus utilities for working with
requests, responses and errors.
"""

import hashlib
from random import random

from urllib import quote
from django.conf import settings
from django.core import urlresolvers
from django.http import HttpResponseRedirect
from django.template import RequestContext
from django.shortcuts import render_to_response
from django.utils.safestring import mark_safe

from apps.gcd.views.pagination import DiggPaginator

from apps.stddata.models import Language
from apps.stats.models import CountStats
from apps.indexer.models import Error

ORDER_ALPHA = "alpha"
ORDER_CHRONO = "chrono"

def index(request):
    """Generates the front index page."""

    stats = CountStats.objects.filter(language__isnull=True,
                                      country__isnull=True)
    language = None
    stats_for_language = []

    # We only want to show English-specific anything if English was
    # explicitly passed in the URL.  Otherwise we use the English-language
    # page but without anything specific to only the English contents of
    # the database.
    code = request.LANGUAGE_CODE
    if code and (code != 'en' or request.GET.get('lang') == 'en'):
        try:
            language = Language.objects.get(code=code)
            stats_for_language = CountStats.objects.filter(language=language)
        except Language.DoesNotExist:
            pass

    if (not code) or code not in settings.FRONT_PAGE_LANGUAGES:
        # Make certain that we pick an actual existing page, but
        # leave the language stats matching what was requested.
        code = 'en'
    base_path = 'managed_content/gcd/%s/' % code

    template_vars = {}
    for template in ('front_page_title', 'front_page_main', 'front_page_lower',
                     'fb_feed'):
        template_vars[template] = '%s/%s.html' % (base_path, template)

    template_vars.update({
        'stats': stats,
        'language': language,
        'stats_for_language': stats_for_language,
        'CALENDAR': settings.CALENDAR,
    })
    return render_to_response('gcd/index.html', template_vars,
                              context_instance=RequestContext(request))

class ResponsePaginator(object):
    """
    Uses DiggPaginator from
    http://bitbucket.org/miracle2k/djutils/src/tip/djutils/pagination.py.
    We could reconsider writing our own code.
    """
    def __init__(self, queryset, vars=None, template=None, per_page=100,
                 callback_key=None, callback=None, view=None):
        """
        Either template or view should be set
        """
        if template is None and view is None:
            raise ValueError
        self.template = template
        self.vars = vars and vars or {}
        self.callback_key = callback_key
        self.callback = callback
        self.view = view
        self.p = DiggPaginator(queryset, per_page, body=7, padding=2, tail=1)

    def paginate(self, request):
        page_num = 1
        self.vars['pagination_type'] = 'num'
        redirect = None
        if ('page' in request.GET):
            try:
                page_num = int(request.GET['page'])
                if page_num > self.p.num_pages:
                    redirect = self.p.num_pages
                elif page_num < 1:
                    redirect = 1
            except ValueError:
                if 'alpha_paginator' in self.vars and \
                  request.GET['page'][:2] == 'a_':
                    self.alpha_paginator = self.vars['alpha_paginator']
                    try:
                        alpha_page_num = int(request.GET['page'][2:])
                        if alpha_page_num > self.alpha_paginator.num_pages:
                            redirect = 'a_%d' % self.alpha_paginator.num_pages
                        elif page_num < 1:
                            redirect = 'a_1'
                        else:
                            alpha_page = \
                              self.alpha_paginator.page(alpha_page_num)
                            self.vars['pagination_type'] = 'alpha'
                            self.vars['alpha_page'] = alpha_page
                            issue_count = self.alpha_paginator.number_offset
                            for i in range(1,alpha_page_num):
                                issue_count += \
                                  self.alpha_paginator.page(i).count
                            page_num = int(issue_count/self.p.per_page) + 1
                    except ValueError:
                        redirect = 1
                else:
                    redirect = 1

        if redirect is not None:
            args = request.GET.copy()
            args['page'] = redirect
            return HttpResponseRedirect(quote(request.path.encode('UTF-8')) +
                                        u'?' + args.urlencode())

        page = self.p.page(page_num)
        self.vars['items'] = page.object_list
        self.vars['paginator'] = self.p
        self.vars['page'] = page
        self.vars['page_number'] = page_num

        if self.callback_key is not None:
            self.vars[self.callback_key] = self.callback(page)

        if self.view:
            return self.view()

        return render_to_response(self.template, self.vars,
                                  context_instance=RequestContext(request))

def paginate_response(request, queryset, template, vars, per_page=100,
                      callback_key=None, callback=None):
    return ResponsePaginator(queryset, vars=vars, template=template,
                             per_page=per_page,
                             callback_key=callback_key,
                             callback=callback).paginate(request)


from django.urls import path, re_path

from . import views

urlpatterns = [
    # 执行ansible script/ playbook
    path('v1/tasks/exec/command', views.ExecCommandApiView.as_view()),

    # 获取脚本和playbook
    path('v1/tasks/scripts_playbook', views.ScriptPlaybookApiView.as_view()),
    path('v1/tasks/scripts_playbook/<str:filename>/', views.ScriptPlaybookApiView.as_view()),
]


"""Load Celery tasks for account submission.

This module is responsible for instantiating the Celery tasks used for account
creation using our special credentials. Other modules should import from here,
rather than from ocflib directly.
"""
from celery import Celery
from django.conf import settings
from ocflib.account.submission import get_tasks as real_get_tasks

celery_app = Celery(
    broker=settings.CELERY_BROKER,
    backend=settings.CELERY_BACKEND,
)
_tasks = real_get_tasks(celery_app)

create_account = _tasks.create_account
validate_then_create_account = _tasks.validate_then_create_account
get_pending_requests = _tasks.get_pending_requests
approve_request = _tasks.approve_request
reject_request = _tasks.reject_request


from torchfusion.gan.learners import *
from torchfusion.gan.applications import StandardGenerator,StandardProjectionDiscriminator
from torch.optim import Adam
from torchfusion.datasets import fashionmnist_loader
import torch.cuda as cuda
import torch.nn as nn
import torch
from torch.autograd import Variable

G = StandardGenerator(output_size=(1,32,32),latent_size=128)
D = StandardProjectionDiscriminator(input_size=(1,32,32))

if cuda.is_available():
    G = nn.DataParallel(G.cuda())
    D = nn.DataParallel(D.cuda())

g_optim = Adam(G.parameters(),lr=0.0002,betas=(0.5,0.999))
d_optim = Adam(D.parameters(),lr=0.0002,betas=(0.5,0.999))

dataset = fashionmnist_loader(size=32,batch_size=64)

class CustomGanLearner(BaseGanCore):

    def train(self,train_loader, gen_optimizer,disc_optimizer,latent_size,loss_fn=nn.BCELoss(),**kwargs):

        self.latent_size = latent_size
        self.loss_fn = loss_fn
        super().__train_loop__(train_loader,gen_optimizer,disc_optimizer,**kwargs)

    def __disc_train_func__(self, data):

        super().__disc_train_func__(data)

        self.disc_optimizer.zero_grad()

        if isinstance(data, list) or isinstance(data, tuple):
            x = data[0]
        else:
            x = data

        batch_size = x.size(0)

        source = self.dist.sample((batch_size,self.latent_size))

        real_labels = torch.ones(batch_size,1)
        fake_labels = torch.zeros(batch_size,1)

        if self.cuda:
            x = x.cuda()
            source = source.cuda()
            real_labels = real_labels.cuda()
            fake_labels = fake_labels.cuda()

        x = Variable(x)
        source = Variable(source)

        outputs = self.disc_model(x)

        generated = self.gen_model(source)
        gen_outputs = self.disc_model(generated.detach())

        gen_loss = self.loss_fn(gen_outputs,fake_labels)

        real_loss = self.loss_fn(outputs,real_labels)

        loss = gen_loss + real_loss
        loss.backward()
        self.disc_optimizer.step()

        self.disc_running_loss.add_(loss.cpu() * batch_size)

    def __gen_train_func__(self, data):

        super().__gen_train_func__(data)

        self.gen_optimizer.zero_grad()

        if isinstance(data, list) or isinstance(data, tuple):
            x = data[0]
        else:
            x = data
        batch_size = x.size(0)

        source = self.dist.sample((batch_size,self.latent_size))

        real_labels = torch.ones(batch_size,1)

        if self.cuda:
            source = source.cuda()
            real_labels = real_labels.cuda()

        source = Variable(source)

        fake_images = self.gen_model(source)
        outputs = self.disc_model(fake_images)

        loss = self.loss_fn(outputs,real_labels)
        loss.backward()

        self.gen_optimizer.step()

        self.gen_running_loss.add_(loss.cpu() * batch_size)

learner = CustomGanLearner(G,D)

if __name__ == "__main__":
    learner.train(dataset,gen_optimizer=g_optim,disc_optimizer=d_optim,model_dir="./mnist-gan-custom-logic",latent_size=128,batch_log=True)


"""Plot pairs."""
from __future__ import annotations

# Standard library
import filecmp
import os
import re
import sys
import traceback
from pathlib import Path
from subprocess import PIPE
from subprocess import Popen
from typing import Iterator
from typing import Optional
from typing import Sequence
from typing import Union

# Local
from .config import RunConfig
from .utils import run_cmd

class PlotPair:
    def __init__(
        self,
        path1: Union[Path, str],
        path2: Union[Path, str],
        base1: Optional[Union[Path, str]] = None,
        base2: Optional[Union[Path, str]] = None,
    ) -> None:
        """Create an instance of ``PlotPair``.

        The two plots must have the same file name but are located in different
        directories, which are part of the paths but may also be specified
        explicitly (or have to; now sure; needs cleanup).

        Args:
            path1: Path to first plot; file name must be the same as ``path2``.

            path2: Path to second plot; file name must be the same as ``path1``.

            base1 (optional): Base directory of ``path1``; equal to the basename
                of ``path1`` plus all preceding subdirectories shared with
                ``path2``; e.g., the bases for two paths 'foo/hello/world.png`
                and 'bar/hello/world.png` would be 'foo` and 'bar`, given the
                shared base compolent 'hello/world.png`.

            base2 (optional): Like ``base1`` but for ``path2``.

        """
        self.path1: Path = Path(path1)
        self.path2: Path = Path(path2)
        self.base1: Path = Path(base1 or self.path1.root)
        self.base2: Path = Path(base2 or self.path2.root)
        shared_base1 = self.path1.relative_to(self.base1)
        shared_base2 = self.path2.relative_to(self.base2)
        if shared_base1 != shared_base2:
            raise ValueError(
                "inconsistent paths and bases; shared path components differ:"
                f" {shared_base1} != {shared_base2}"
            )
        self.shared_base: Path = shared_base1
        self.shared_root: Path = Path(os.path.commonpath([self.base1, self.base2]))

    @property
    def rel_path1(self) -> Path:
        return self.path1.relative_to(self.shared_root)

    @property
    def rel_path2(self) -> Path:
        return self.path2.relative_to(self.shared_root)

    def create_diff(
        self,
        diffs_path: Optional[Union[Path, str]],
        cfg: RunConfig,
        raw: bool = False,
    ) -> Optional[Path]:
        _name_ = f"{__name__}.{type(self).__name__}.create_diff"
        if cfg.debug:
            print(f"DBG:{_name_}: comparing pair {self.shared_base}")
        if filecmp.cmp(self.path1, self.path2):
            if cfg.verbose:
                print(f"identical: {self.shared_base}")
            return None
        else:
            if cfg.verbose:
                print(f"differing: {self.shared_base}")
            diff_path = self.shared_base
            if diffs_path:
                diff_path = Path(diffs_path) / diff_path
            if raw:
                # Add '-raw' before suffix, e.g., a.png -> a-raw.png
                diff_path = diff_path.parent / Path(
                    re.sub(r"(.\w+$)", r"-raw\1", diff_path.name)
                )
            diff_path.parent.mkdir(parents=True, exist_ok=True)
            if self._equal_sized():
                self._compare_equal_sized(diff_path, cfg, raw)
            else:
                self._compare_unequal_sized(diff_path, cfg, raw)
            return diff_path

    def _compare_equal_sized(self, diff_path: Path, cfg: RunConfig, raw: bool) -> None:
        _name_ = f"{__name__}.{type(self).__name__}._compare_equal_sized"
        cmd_args = ["compare", str(self.path1), str(self.path2)]
        if raw:
            cmd_args += ["-compose src -highlight-color red"]
        cmd_args += [str(diff_path)]
        if cfg.debug:
            print(
                f"DBG:{_name_}: create diff plot with following command:\n$ "
                + " \\\n    ".join(cmd_args)
            )
        cmd_args = [sub_arg for arg in cmd_args for sub_arg in arg.split()]
        run_cmd(cmd_args)

    # pylint: disable=R0914  # too-many-locals (>15)
    def _compare_unequal_sized(
        self, diff_path: Path, cfg: RunConfig, raw: bool
    ) -> None:
        _name_ = f"{__name__}.{type(self).__name__}._compare_unequal_sized"
        w1 = int(self._identify(self.path1, fmt="%[w]", trim=True))
        w2 = int(self._identify(self.path2, fmt="%[w]", trim=True))
        h1 = int(self._identify(self.path1, fmt="%[h]", trim=True))
        h2 = int(self._identify(self.path2, fmt="%[h]", trim=True))
        width = max(w1, w2)
        height = max(h1, h2)
        size = f"{width}x{height}"
        if cfg.debug:
            print(f"DBG:{_name_}: original sizes: ({w1}x{h1}), ({w2}x{h2})")
            print(f"DBG:{_name_}: target size: ({size})")
        args_prep = (
            f"-trim -resize {size} -background white -gravity north-west -extent {size}"
            f" -bordercolor white -border 10"
        )
        args_path1 = f"( {args_prep} {str(self.path1)} )"
        args_path2 = f"( {args_prep} {str(self.path2)} )"
        cmd_prep = f"convert {args_path1} {args_path2} miff:-"
        cmd_comp = "compare miff:-"
        if raw:
            cmd_comp += " -compose src -highlight-color red"
        cmd_comp += f" {diff_path}"
        cmd = " | ".join([cmd_prep, cmd_comp])
        if cfg.debug:
            print(f"DBG:{_name_}: creating diff plot with following command:\n$ {cmd}")
        with Popen(
            cmd_comp.split(),
            stdin=Popen(cmd_prep.split(), stdout=PIPE).stdout,
            stdout=PIPE,
        ) as p:
            _, stderr = p.communicate()
        if stderr:
            raise RuntimeError(
                f"error running command '{cmd}':\n{stderr.decode('ascii')}"
            )

    def _equal_sized(self) -> bool:
        """Determine whether the images are of equal size."""
        size1 = self._identify(self.path1, fmt="%[w]x%[h]")
        size2 = self._identify(self.path2, fmt="%[w]x%[h]")
        return size1 == size2

    @staticmethod
    def _identify(path: Path, fmt: Optional[str] = None, trim: bool = False) -> str:
        """Run IM's identify for ``path``."""
        cmd_idfy = "identify"
        if fmt is not None:
            cmd_idfy += f" -format {fmt}"
        if not trim:
            cmd_idfy += f" {path}"
            with Popen(cmd_idfy.split(), stdout=PIPE) as p:
                stdout, _ = p.communicate()
                return stdout.decode("ascii")
        cmd_idfy += " miff:-"
        cmd_trim = f"convert -trim {path} miff:-"
        cmd = " | ".join([cmd_trim, cmd_idfy])
        with Popen(
            cmd_idfy.split(),
            stdin=Popen(cmd_trim.split(), stdout=PIPE).stdout,
            stdout=PIPE,
        ) as p:
            stdout, stderr = p.communicate()
            if stderr:
                raise RuntimeError(
                    f"error running command '{cmd}':\n{stderr.decode('ascii')}"
                )
            return stdout.decode("ascii")

class PlotPairSequence:
    """A sequence of ``PlotPair`` instances."""

    # pylint: disable=R0913  # too-many-arguments (>5)
    def __init__(
        self,
        paths1: Sequence[Path],
        paths2: Sequence[Path],
        base1: Optional[Path],
        base2: Optional[Path],
        sort: bool = True,
    ) -> None:
        """Create an instance of ``PlotPair``."""
        paths1 = list(paths1)
        paths2 = list(paths2)
        self.missing1: list[Path]
        self.missing2: list[Path]
        self.missing1, self.missing2 = self.check_paths_equiv(
            paths1=paths1,
            paths2=paths2,
            base1=base1,
            base2=base2,
            err_action=None,
            del_missing=True,
            sort_rel=sort,
        )
        self.pairs: list[PlotPair] = [
            PlotPair(path1=old_path, path2=new_path, base1=base1, base2=base2)
            for old_path, new_path in zip(paths1, paths2)
        ]

    def create_diffs(
        self,
        diffs_path: Path,
        cfg: RunConfig,
        *,
        err_ok: bool = False,
        raw: bool = False,
    ) -> list[Path]:
        """Create difference plots for those pairs that differ.

        Args:
            diffs_path: Path where diff plots are saved.

            cfg: Run configuration.

            err_ok (optional): Continue in case of an error instead of raising
                an exception.

            raw (optional): Only return raw diff mask.

        """
        if len(self) == 0:
            print("warning: no pairs of plots to create diff plots")
            return []
        print(f"compare {len(self)} pairs of plots")
        diff_paths: list[Path] = []
        for pair in self:
            try:
                diff_path = pair.create_diff(diffs_path, cfg, raw=raw)
            # pylint: disable=W0703  # broad-except
            except Exception as e:
                if not err_ok:
                    raise Exception(
                        f"error comparing {pair.rel_path1} and {pair.rel_path2}:\n{e}"
                    ) from e
                else:
                    print("-" * 50, file=sys.stderr)
                    traceback.print_exc()
                    print("-" * 50, file=sys.stderr)
                    print(
                        "error during diff creation (see traceback above);"
                        f" abort comparison of {pair.rel_path1} and {pair.rel_path2}",
                        file=sys.stderr,
                    )
            if diff_path is not None:
                diff_paths.append(diff_path)
        return diff_paths

    def create_composite_diff(self, diffs_path: Path, cfg: RunConfig) -> Path:
        """Create composite difference plot for those pairs that differ.

        Args:
            diffs_path: Path where diff plots are saved.

            cfg: Run configuration.

        """
        _name_ = f"{__name__}.{type(self).__name__}.create_composite_diff"
        diff_paths = self.create_diffs(diffs_path, cfg, err_ok=False, raw=True)
        if not diff_paths:
            raise Exception("missing diff plots to create composite diff plot")
        composite_path = diffs_path / f"composite_diff_{len(diff_paths)}x.png"
        cmd_args = (
            ["composite"]
            + list(map(str, diff_paths))
            + ["-compose src", str(composite_path)]
        )
        if cfg.verbose:
            print(f"create composite of {len(diff_paths)} diff plots: {composite_path}")
        if cfg.debug:
            print(
                f"DBG:{_name_}: create composite raw diff plot with following command:"
                + ("\n$ " + " \\\n    ".join(cmd_args))
            )
        cmd_args = [sub_arg for arg in cmd_args for sub_arg in arg.split()]
        try:
            run_cmd(cmd_args)
        # pylint: disable=W0703  # broad-except
        except Exception as e:
            raise Exception(
                f"error creating composite diff plot {composite_path}:\n{e}"
            ) from e
        if cfg.verbose:
            print(f"remove {len(diff_paths)} raw diff plots")
        for path in diff_paths:
            if cfg.debug:
                print(f"DBG:{_name_}: remove {path}")
            path.unlink()
        return composite_path

    def __iter__(self) -> Iterator[PlotPair]:
        return iter(self.pairs)

    def __len__(self) -> int:
        return len(self.pairs)

    # pylint: disable=R0913  # too-many-arguments (>5)
    @staticmethod
    def check_paths_equiv(
        paths1: list[Path],
        paths2: list[Path],
        base1: Optional[Path] = None,
        base2: Optional[Path] = None,
        *,
        err_action: Optional[str] = "raise",
        del_missing: bool = False,
        sort_rel: bool = False,
    ) -> tuple[list[Path], list[Path]]:
        """Check that to collections of paths are equivalent.

        Args:
            paths1: First collection of paths.

            paths2: Second collection of paths.

            base1 (optional): Base paths subtracted from ``paths1`` to obtain
                relative paths.

            base2 (optional): Base paths subtracted from ``paths2`` to obtain
                relative paths.

            err_action (optional): Action to take when a path in one collection
                is missing in the other; "raise" an exception; only "warn" the
                user; or do nothing (None).

            del_missing (optional): Delete paths from the collection if they are
                missing in the other; incompatible with ``action`` "raise".

            sort_rel (optional): Sort the paths by their relative
                representation; if ``base1`` or ``base2`` is omitted, the
                respective paths are sorted as is.

        Returns:
            Paths missing in ``path1`` and ``paths2``, respectively.

        """
        actions = ["raise", "warn", None]
        if err_action not in actions:
            raise ValueError(f"invalid action '{err_action}'; must be among {actions}")
        if del_missing and err_action == "raise":
            raise ValueError("del_missing=T is incompatible with action 'raise'")

        def subtract_base(paths: list[Path], base: Optional[Path]) -> list[Path]:
            if base is None:
                return list(paths)
            return [path.relative_to(base) for path in paths]

        # pylint: disable=R0913  # too-many-arguments (>5)
        def run(
            name1: str,
            paths1: list[Path],
            base1: Optional[Path],
            name2: str,
            paths2: list[Path],
            base2: Optional[Path],
        ) -> list[Path]:
            rel_paths1 = subtract_base(paths1, base1)
            rel_paths2 = subtract_base(paths2, base2)
            missing2: list[Path] = []
            for path1, rel_path1 in zip(list(paths1), list(rel_paths1)):
                if rel_path1 in rel_paths2:
                    continue
                missing2.append(rel_path1)
                msg = f"path from '{name1}' missing in '{name2}': {rel_path1}"
                if err_action == "raise":
                    raise AssertionError(msg)
                elif err_action == "warn":
                    print(f"warning: {msg}", file=sys.stderr)
                elif err_action is None:
                    # Do nothing
                    pass
                if del_missing:
                    paths1.remove(path1)
                    rel_paths1.remove(rel_path1)
            if sort_rel:
                paths1_sorted_by_rel = sorted(
                    [(rel_path1, path1) for path1, rel_path1 in zip(paths1, rel_paths1)]
                )
                paths1.clear()
                for _, path1 in paths1_sorted_by_rel:
                    paths1.append(path1)
            return missing2

        missing1 = run("paths2", paths2, base2, "paths1", paths1, base1)
        missing2 = run("paths1", paths1, base1, "paths2", paths2, base2)
        return (missing1, missing2)


"""
Fit Gaussian Mixture by YUAN Ruihong
"""

import argparse
import re
from random import random

import numpy as np
import pandas as pd
from scipy.stats import norm

from labw_utils import UnmetDependenciesError
from labw_utils.commonutils.stdlib_helper.argparse_helper import ArgumentParserWithEnhancedFormatHelp
from labw_utils.commonutils.stdlib_helper.logger_helper import get_logger
from labw_utils.mlutils.ndarray_helper import describe
from labw_utils.typing_importer import List
from yasim.helper.gmm import GaussianMixture1D
from yasim.helper.plot_utils import plot

try:
    import pytest

    _ = pytest.importorskip("pyarrow")
except ImportError:
    pytest = None
    try:
        import pyarrow
    except ImportError as e:
        raise UnmetDependenciesError("pyarrow") from e

_lh = get_logger()

def create_parser() -> argparse.ArgumentParser:
    parser = ArgumentParserWithEnhancedFormatHelp(
        prog="python -m yasim_scripts fit_gmm",
        description=__doc__.splitlines()[1]
    )
    parser.add_argument(
        'data',
        help="Dataset in Apache Parquet Format"
    )
    parser.add_argument(
        '-c', '--num_components', type=int, required=False, default=2
    )
    parser.add_argument(
        '--col_regex', type=str, required=False, default="SRR.*"
    )
    parser.add_argument(
        '--filter_zero', action="store_true"
    )
    parser.add_argument(
        '--scale', action="store_true"
    )
    parser.add_argument(
        '--num_iters', type=int, required=False, default=100
    )
    return parser

def main(args: List[str]):
    args = create_parser().parse_args(args)
    _lh.info("GMM: Processing data...")
    if args.data == 'dummy':
        mu1, sigma1 = random() * 100, random() * 10
        mu2, sigma2 = random() * 100, random() * 10
        p = random()
        model1 = norm(mu1, sigma1)
        model2 = norm(mu2, sigma2)
        data = np.append(model1.rvs(size=int(p * 200)), model2.rvs(size=int((1 - p) * 200)))
        print(f"Ground truth is {p}*N({mu1}, {sigma1}) + {1 - p}*N({mu2}, {sigma2})")
    else:
        df = pd.read_parquet(args.data)
        col_names = df.columns[list(map(lambda x: re.match(args.col_regex, x) is not None, df.columns))]
        df: pd.DataFrame = df[col_names]
        df = df.fillna(0)
        if args.scale:
            df = df.apply(
                lambda x: np.log10(x / max(x.max(), 1E-9) * 5000 + 1),
                axis="columns"
            )

        data = np.asarray(df.to_numpy()).ravel()
        if args.filter_zero:
            data = data[data != 0]
    print(describe(data))
    # plot(data)

    _lh.info("GMM: Fitting START")
    # auto_select_num_components(data, 2, 8, args.num_iters)
    model = GaussianMixture1D(
        n_components=args.num_components,
        n_iter=args.num_iters
    ).fit(data)
    _lh.info("GMM: Fitting FIN")

    plot(data, model)
    print(model.export())


import requests
from bs4 import BeautifulSoup
import codecs
import html
import os.path
import errno

user_agent = 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/36.0.1985.143 Safari/537.36'
headers = {'User-Agent': user_agent}
url = 'https://gall.dcinside.com/mgallery/board/view/?id=lastorigin&no=1384131'
url_get = requests.get(url, headers=headers)
html_code = url_get.content.decode('utf-8')
soup = BeautifulSoup(html_code, 'html.parser')
established_date = soup.find('span', {'class' : 'gall_date'}).get('title')[0:10]
title_article = soup.find('span', {'class' : 'title_subject'}).getText()
contents_article = soup.find('div', {'class' : 'writing_view_box'})
folder_name = established_date + ' ' + title_article

try:
    if not(os.path.isdir(folder_name)):
        os.makedirs(os.path.join(folder_name))
except OSError as e:
    if e.errno != errno.EEXIST:
        print("Failed to create directory")
        raise

filepath = os.path.join(folder_name, 'contents.html')
with codecs.open(filepath, 'w', encoding='utf-8') as f:
    f.write(str(contents_article))


import sys
import spacy

nlp = spacy.load("en_core_web_sm")
merge_nouns = nlp.create_pipe("merge_noun_chunks")
merge_ents = nlp.create_pipe("merge_entities")
nlp.add_pipe(merge_nouns)
nlp.add_pipe(merge_ents)

if __name__ == "__main__":
    with open(sys.argv[1]) as f:
        text = f.readlines()

    for line in text:
        s = nlp(line)
        print(" ".join([str((t.text, t.dep_)) for t in s]))