#!/usr/bin/env python
# coding: utf-8

# In[ ]:

import pandas as pd
import requests
from bs4 import BeautifulSoup
import time
import os

Intro='안녕하세요. 웹툰 선호도를 조사 중입니다.'
maker='wony'
BASE_DIR=os.getcwd()

class crl():

    def __init__(self, base_dir):
        self.base_dir=BASE_DIR

    def get_naverToon_info():

        naverToon_url="https://comic.naver.com/webtoon/creation.nhn"
        naver_resp=requests.get(naverToon_url)
        naver_soup=BeautifulSoup(naver_resp.content,'html.parser')
        all_list=naver_soup.find('div', class_="all_list").find_all('li')
    #     start=time.time()
        all_info=[]
        for item in all_list:
            a_tag=item.find('a')
            title=a_tag['title']

            link=a_tag['href']
            item_resp=requests.get("https://comic.naver.com/"+link)
            item_soup=BeautifulSoup(item_resp.content,'html.parser')

            toon_info=item_soup.find('div', class_="webtoon")
            if item_soup.find('div', class_="webtoon").find('ul',class_="category_tab"):
                day=toon_info.find('ul',class_="category_tab").find('li',class_="on").text
            else:
                day_temp=item_soup.find('table', class_="viewList").find('td',class_="num").text.split('.')
                day_temp[2]='완결'
                day=" ".join(day_temp)

            author_info=toon_info.find('span',class_='wrt_nm')
            author=author_info.text.strip()

    #         detail_info=toon_info.find('p')
    #         detail=detail_info.text.decode('utf-8')

    #         all_info.append([title,author,day,detail])

            print('{}에 대한 정보를 가져오는 중입니다.'.format(title))
            df=pd.DataFrame({'제목':[title],'작가':[author],'연재상태':[day]})
            if all_list.index(item)==0:
                df.to_csv('all_info.csv', index=False, columns=['제목','작가','연재상태'],encoding='cp949')

            else:
                df.to_csv('all_info.csv', index=False, header=False,encoding='cp949',mode='a')
        print("모든 웹툰 정보가 저장되었습니다.")
    #     df_all_info=pd.DataFrame(all_info, columns=['제목','작가','연재상태','상세설명'])
    #         all_info.append([title,author,day])

    #     df_all_info=pd.DataFrame(all_info, columns=['제목','작가','연재상태'])
    #     df_all_info.to_csv('all_info_detail.csv', encoding='cp949')

    #         print("Runtime: %0.3f"%(time.time() - start))
    def pick(*args):# 0<args<839
        pick_toon=pd.read_csv('all_info.csv', encoding='cp949')

    #     pick_toon=pd.DataFrame(pick_toon, columns=['제목','작가','연재상태'])
        pick_title=[]
        for arg in args:
            pick_title.append(pick_toon.loc[arg-1,'제목'])
    #     return pick_toon
        return pick_title

    # pick_title=pick(1,11,445)
    def find_genre(title):
        genre_url='https://comic.naver.com/webtoon/genre.nhn'
        genre_list=['episode','omnibus','story','daily','comic','fantasy','action','drama','pure','sensibility','thrill','historical','sports']
        title_genre=[]
        for genre in genre_list:
            params={
                'genre': genre
            }
            genre_resp=requests.get(genre_url, params=params)
            genre_soup=BeautifulSoup(genre_resp.content,'html.parser')
    #         for title in pick_title:
    #             print("{}은 {}장르입니다.".format(title, genre)
            if genre_soup.find_all('a',text=title):
                title_genre.append(genre)
            else:
                pass
        return title_genre


from PyQt5.QtWidgets import QApplication, QWidget, QVBoxLayout, QPushButton, QFileDialog, QTextEdit, QLabel, QComboBox, QSizePolicy, QLineEdit, QMessageBox, QPlainTextEdit
from collections import deque
import sys
import os
import shutil
import pickle

class MyApp(QWidget):
    def __init__(self):
        super().__init__()
        self.initUI()

    def initUI(self):
        # Initialize variables for source and target folders
        self.source_folder = ''
        self.target_folder = ''

        # List of recently used folders
        self.recent_folders = self.load_recent_folders()

        # User interface setup
        vbox = QVBoxLayout()
        self.setLayout(vbox)

        # Button to select source folder
        self.source_button = QPushButton('Select Source Folder', self)
        self.source_button.clicked.connect(self.select_source)
        self.source_button.setSizePolicy(QSizePolicy.Preferred, QSizePolicy.Expanding)
        vbox.addWidget(self.source_button)

        # Text field to display the path of the source folder
        self.source_folder_label = QLineEdit(self)
        self.source_folder_label.setReadOnly(True)
        vbox.addWidget(self.source_folder_label)

        # Combo box for recently used source folders
        self.source_recent = QComboBox(self)
        self.source_recent.addItem('Recent Folders')
        self.source_recent.addItems(self.recent_folders)
        self.source_recent.currentIndexChanged.connect(self.select_source_recent)
        self.source_recent.setSizePolicy(QSizePolicy.Preferred, QSizePolicy.Expanding)
        vbox.addWidget(self.source_recent)

        # Button to select target folder
        self.target_button = QPushButton('Select Target Folder', self)
        self.target_button.clicked.connect(self.select_target)
        self.target_button.setSizePolicy(QSizePolicy.Preferred, QSizePolicy.Expanding)
        vbox.addWidget(self.target_button)

        # Text field to display the path of the target folder
        self.target_folder_label = QLineEdit(self)
        self.target_folder_label.setReadOnly(True)
        vbox.addWidget(self.target_folder_label)

        # Combo box for recently used target folders
        self.target_recent = QComboBox(self)
        self.target_recent.addItem('Recent Folders')
        self.target_recent.addItems(self.recent_folders)
        self.target_recent.currentIndexChanged.connect(self.select_target_recent)
        self.target_recent.setSizePolicy(QSizePolicy.Preferred, QSizePolicy.Expanding)
        vbox.addWidget(self.target_recent)

        # Text field to enter keywords
        self.keyword_label = QLabel('Keywords (separated by newline):', self)
        vbox.addWidget(self.keyword_label)

        self.keyword_entry = QTextEdit(self)
        self.keyword_entry.setSizePolicy(QSizePolicy.Preferred, QSizePolicy.Expanding)
        vbox.addWidget(self.keyword_entry)

        # Combo box to choose whether to move or copy files
        self.operation = QComboBox(self)
        self.operation.addItem('Move Files')
        self.operation.addItem('Copy Files')
        self.operation.setSizePolicy(QSizePolicy.Preferred, QSizePolicy.Expanding)
        vbox.addWidget(self.operation)

        # Button to start file processing
        self.process_button = QPushButton('Process Files', self)
        self.process_button.clicked.connect(self.process_files)
        self.process_button.setSizePolicy(QSizePolicy.Preferred, QSizePolicy.Expanding)
        vbox.addWidget(self.process_button)

        # Text field to display the list of processed files
        self.log_label = QLabel('Processed Files:', self)
        vbox.addWidget(self.log_label)

        self.log = QPlainTextEdit(self)
        self.log.setReadOnly(True)
        vbox.addWidget(self.log)

        # Set the title and size of the window, and display the window
        self.setWindowTitle('File Moving Program')
        self.setGeometry(300, 300, 500, 400)
        self.show()

    def load_recent_folders(self):
        # Load the list of recently used folders
        try:
            with open('recent_folders.pickle', 'rb') as f:
                return pickle.load(f)
        except FileNotFoundError:
            return deque(maxlen=5)

    def save_recent_folders(self):
        # Save the list of recently used folders
        with open('recent_folders.pickle', 'wb') as f:
            pickle.dump(self.recent_folders, f)

    def add_recent_folder(self, folder):
        # Add a folder to the list of recently used folders
        if folder in self.recent_folders:
            self.recent_folders.remove(folder)
        self.recent_folders.appendleft(folder)
        self.save_recent_folders()

    def select_source(self):
        # Select the source folder
        self.source_folder = QFileDialog.getExistingDirectory(self, 'Select Source Folder')
        if self.source_folder:
            self.source_folder_label.setText(self.source_folder)
            self.add_recent_folder(self.source_folder)
            self.source_recent.clear()
            self.source_recent.addItem('Recent Folders')
            self.source_recent.addItems(self.recent_folders)

    def select_source_recent(self, index):
        # Select a recently used source folder
        if index > 0:
            self.source_folder = self.recent_folders[index-1]
            self.source_folder_label.setText(self.source_folder)

    def select_target(self):
        # Select the target folder
        self.target_folder = QFileDialog.getExistingDirectory(self, 'Select Target Folder')
        if self.target_folder:
            self.target_folder_label.setText(self.target_folder)
            self.add_recent_folder(self.target_folder)
            self.target_recent.clear()
            self.target_recent.addItem('Recent Folders')
            self.target_recent.addItems(self.recent_folders)

    def select_target_recent(self, index):
        # Select a recently used target folder
        if index > 0:
            self.target_folder = self.recent_folders[index-1]
            self.target_folder_label.setText(self.target_folder)

    def process_files(self):
        # Process the files
        try:
            keywords = self.keyword_entry.toPlainText().split('\n')
            # If no keywords are entered, the function is exited.
            if not any(keywords):
                QMessageBox.information(self, "Notice", "Please enter keywords.")
                return
            processed_files = []
            for file_name in os.listdir(self.source_folder):
                if any(keyword in file_name for keyword in keywords if keyword):
                    source_file = os.path.join(self.source_folder, file_name)
                    target_file = os.path.join(self.target_folder, file_name)
                    # Move or copy the file
                    if self.operation.currentText() == 'Move Files':
                        shutil.move(source_file, target_file)
                    else:
                        shutil.copy2(source_file, target_file)
                    processed_files.append(file_name)
            if processed_files:
                # Display the list of processed files
                self.log.appendPlainText(f"\n{len(processed_files)} files have been {self.operation.currentText()}:\n" + "\n\n".join(processed_files))
                QMessageBox.information(self, "Success", f"{len(processed_files)}File(s) processing is complete.")
            else:
                QMessageBox.information(self, "Notice", "No files match the keywords.")
        except Exception as e:
            QMessageBox.critical(self, "Error", str(e))

if __name__ == '__main__':
    app = QApplication(sys.argv)
    ex = MyApp()
    sys.exit(app.exec_())


import numpy as np
import matplotlib.pyplot as plt
from mpl_toolkits.mplot3d import Axes3D
from matplotlib.animation import FuncAnimation

class RotationalMatrix3D:
    def __init__(self, roll, pitch, yaw):
        self.roll = roll
        self.pitch = pitch
        self.yaw = yaw

    def __mul__(self, vector):
        """
        Applies the rotational matrix to a 3D vector, list, or tuple.
        Returns the transformed vector as a NumPy array.
        """
        R_roll = np.array([[1, 0, 0],
                           [0, np.cos(self.roll), -np.sin(self.roll)],
                           [0, np.sin(self.roll), np.cos(self.roll)]])

        R_pitch = np.array([[np.cos(self.pitch), 0, np.sin(self.pitch)],
                            [0, 1, 0],
                            [-np.sin(self.pitch), 0, np.cos(self.pitch)]])

        R_yaw = np.array([[np.cos(self.yaw), -np.sin(self.yaw), 0],
                          [np.sin(self.yaw), np.cos(self.yaw), 0],
                          [0, 0, 1]])

        R = np.dot(R_roll, np.dot(R_pitch, R_yaw))

        return np.dot(R, vector)

    def visualize(self, vector):
        """
        Visualizes the original and transformed vectors using matplotlib's quiver3D plot.
        The original vector is plotted in blue, and the transformed vector is plotted in red.
        """
        transformed_vector = self.__mul__(vector)

        fig = plt.figure()
        ax = fig.add_subplot(111, projection='3d')

        ax.quiver(0, 0, 0, vector[0], vector[1], vector[2], color='blue', label='Original')
        ax.quiver(0, 0, 0, transformed_vector[0], transformed_vector[1], transformed_vector[2], color='red', label='Transformed')

        ax.set_xlim([-5, 5])
        ax.set_ylim([-5, 5])
        ax.set_zlim([-5, 5])
        ax.set_xlabel('X')
        ax.set_ylabel('Y')
        ax.set_zlabel('Z')
        ax.set_title('3D Rotational Matrix Transformation')
        ax.legend()
        plt.grid(True)
        plt.show()

    def animate(self, vector, duration=10, frame_rate=30):
        """
        Animates the rotation from the original position to the transformed position.
        The animation lasts for the specified duration in seconds.
        """
        num_frames = int(duration * frame_rate)
        angles = np.linspace(0, 2 * np.pi, num_frames)

        fig = plt.figure()
        ax = fig.add_subplot(111, projection='3d')

        q = ax.quiver(0, 0, 0, vector[0], vector[1], vector[2], color='blue', label='Original')
        q_animation = ax.quiver(0, 0, 0, vector[0], vector[1], vector[2], color='red', label='Transformed')

        def update_frame(frame):
            roll = self.roll * angles[frame] / (2 * np.pi)
            pitch = self.pitch * angles[frame] / (2 * np.pi)
            yaw = self.yaw * angles[frame] / (2 * np.pi)

            R = RotationalMatrix3D(roll, pitch, yaw)

            transformed_vector = R * vector

            q_animation.set_segments([(np.array([0, 0, 0]), transformed_vector)])

            return q_animation

        animation = FuncAnimation(fig, update_frame, frames=num_frames, interval=1000 / frame_rate, blit=False)

        ax.set_xlim([-5, 5])
        ax.set_ylim([-5, 5])
        ax.set_zlim([-5, 5])
        ax.set_xlabel('X')
        ax.set_ylabel('Y')
        ax.set_zlabel('Z')
        ax.set_title('3D Rotational Matrix Transformation Animation')
        ax.legend()

        plt.grid(True)
        plt.show()

roll = np.pi / 4
pitch = np.pi / 3
yaw = np.pi / 6

vector = np.array([1, 2, 3])

rot_matrix = RotationalMatrix3D(roll, pitch, yaw)
rot_matrix.animate(vector, duration=10, frame_rate=30)


from pyspark.sql import SparkSession
from pyspark.sql.functions import col, collect_list, concat_ws, lit, concat

# Create a Spark session
spark = SparkSession.builder.getOrCreate()

input_path = "hdfs://Master:9000/pig/output/part-r-00000"
output_path = "hdfs://Master:9000/spark/ex1.csv"

# Read data into DataFrame
data = spark.read.format("csv").load(input_path)

# Set column names for DataFrame
column_names = ["beer_ABV", "beer_beerID", "beer_name", "beer_style", "review_overall"]
renamedData = data.toDF(*column_names)

# Select the required columns and cast data types
A1 = renamedData.select(
    col("beer_ABV").cast("double"),
    col("beer_style").cast("string"),
    col("review_overall").cast("double")
)

# Group by beer_style
B1 = A1.groupBy("beer_style")

# Process each group in B1
C1_process = B1.applyInPandas(
        lambda group, rows: rows.sort_values("review_overall", ascending=False).head(5),
        schema="beer_ABV double, beer_style string, review_overall double"
    )

# Gom các giá trị từ nhiều hàng vào một hàng
C1_group = C1_process.groupby("beer_style").agg(
    collect_list(
        concat(
            lit("( "),
            concat_ws(",", C1_process["beer_ABV"], C1_process["beer_style"], C1_process["review_overall"]),
            lit(" )")
        )
    ).alias("groups")
)

C1_tranform = C1_group.withColumn("groups", concat_ws(",", "groups"))

C1 = C1_tranform.select("beer_style", "groups")

# Write the grouped data to CSV file
C1.write.format("csv").option("header", "true").mode("overwrite").option("path", output_path).save()


from django.shortcuts import render
from Blog.models import Post, user, Category
from Blog.forms import PostForm, CategoryForm
from django.contrib.auth.forms import UserCreationForm
from django.contrib import messages
from django.contrib.auth import login, logout, authenticate
from django.contrib.auth.decorators import login_required
from django.urls import reverse_lazy
from django.shortcuts import redirect, render
from django.views.generic import DetailView
from django.http import HttpResponse

# from AdminPanel.forms import PostForm

# Create your views here.
# admin CRUD
@login_required(login_url='login')
def AdminPanel(request):
    if request.user.is_superuser:
        return render(request, 'admin/admin_panel.html')
    else:
        return HttpResponse("
Sorry! you have no permission to access page.
")

def adminPosts(request):
    if request.user.is_superuser:
        posts = Post.objects.all()
        context = {"object_list": posts}
        return render(request, 'admin/posts.html', context)
    else:
        return HttpResponse("
Sorry! you have no permission to access page.
")

def addPost(request):
    if request.user.is_superuser:
        if request.method == 'POST':
            post_form = PostForm(request.POST, request.FILES)
            if post_form.is_valid():
                post_form.save()
                return redirect('Admin_posts')
        post_form = PostForm()
        context = {'form': post_form}
        return render(request, 'admin/add_post.html', context)
    else:
        return HttpResponse("
Sorry! you have no permission to access page.
")

def editPost(request, post_id):
    if request.user.is_superuser:
        post = Post.objects.get(id=post_id)
        post_form = PostForm(instance=post)

        if request.method == 'POST':
            post_form = PostForm(request.POST, instance=post)
            if post_form.is_valid():
                post_form.save()
                return redirect('Admin_posts')
        context = {'form': post_form}
        return render(request, 'admin/add_post.html', context)
    else:
        return HttpResponse("
Sorry! you have no permission to access page.
")

def postDel(request, post_id):
    if request.user.is_superuser:
        return render(request, 'admin/admin_panel.html')
    post = Post.objects.get(id=post_id)
    post.delete()
    return redirect('Admin_posts')

###########

def adminUsers(request):
    if request.user.is_superuser:
        users = user.objects.all()
        context = {"object_list": users}
        return render(request, 'admin/users.html', context)
    else:
        return HttpResponse("
Sorry! you have no permission to access page.
")

def userBlock(request, user_id):
    if request.user.is_superuser:
        user1 = user.objects.get(id=user_id)
        user1.is_active = not user1.is_active
        user1.save()
        return redirect('admin_users')
    else:
        return HttpResponse("
Sorry! you have no permission to access page.
")

def userPromote(request, user_id):
    if request.user.is_superuser:
        user1 = user.objects.get(id=user_id)
        user1.is_superuser = not user1.is_superuser
        user1.save()
        return redirect('admin_users')
    else:
        return HttpResponse("
Sorry! you have no permission to access page.
")

########## categories

def adminCategories(request):
    if request.user.is_superuser:
        cats = Category.objects.all()
        context = {"object_list": cats}
        return render(request, 'admin/categories.html', context)
    else:
        return HttpResponse("
Sorry! you have no permission to access page.
")

def addCategory(request):
    if request.user.is_superuser:
        if request.method == 'POST':
            category_form = CategoryForm(request.POST)
            if category_form.is_valid():
                category_form.save()
                return redirect('categ-page')
        category_form = CategoryForm()
        context = {'form': category_form}
        return render(request, 'admin/add_category.html', context)
    else:
        return HttpResponse("
Sorry! you have no permission to access page.
")

def editCategory(request, category_id):
    if request.user.is_superuser:
        category = Category.objects.get(id=category_id)
        categotyForm = CategoryForm(instance=category)

        if request.method == 'POST':
            categotyForm = CategoryForm(request.POST, instance=category)
            if categotyForm.is_valid():
                categotyForm.save()
                return redirect('categ-page')
        context = {'form': categotyForm}
        return render(request, 'admin/add_category.html', context)

def categoryDelete(request, category_id):
    if request.user.is_superuser:
        category = Category.objects.get(id=category_id)
        category.delete()
        return redirect('categ-page')
    else:
        return HttpResponse("
Sorry! you have no permission to access page.
")

class ArticleDetailView(DetailView):
    model = Post
    template_name = 'admin/article_details.html'


import dominate

from collections import OrderedDict

from dominate.tags import *
from contextlib import suppress
from inflection import humanize, titleize

# Create your services here.

TEXT_GENERAL = 'General'

def json_arrays_destroy(sources):
    """
    Transform json arrays to dictionaries in object
    :param sources:
    :return:
    """
    for element, value in sources.copy().items():
        # If element is dict inspect each key value and clear useless keys
        if isinstance(sources[element], dict):
            json_arrays_destroy(sources[element])

        # If element is list inspect each key value and clear useless keys
        if isinstance(sources[element], list):
            sources.pop(element, None)

    return sources

def json_arrays_to_dictionary(sources):
    """
    Transform json arrays to dictionaries in object
    :param sources:
    :return:
    """
    for element, value in sources.copy().items():
        # If element is int convert to string
        if isinstance(sources[element], int):
            sources[element] = str(sources[element])

        # If element is None convert to string
        if sources[element] is None:
            sources[element] = ""

        # If element is dict inspect each key value and clear useless keys
        if isinstance(sources[element], dict):
            json_arrays_to_dictionary(sources[element])

        # If element is list inspect each key value and clear useless keys
        if isinstance(sources[element], list):
            # Clear of empty objects
            sources[element] = [elementor for elementor in sources[element] if elementor]

            # Loop through each item and create a dictionary
            for index, elements in enumerate(sources[element]):
                indexing = index
                keying = f'{element}_{indexing + 1}'
                sources[keying] = elements

                # Format if integer
                if isinstance(elements, int):
                    sources[element][index] = str(elements)

                # Ignored else for clear viewing
                if not isinstance(elements, int):
                    json_arrays_to_dictionary(elements)

    return sources

def builder(sources, document, etable=None, erows=None):
    """
    Build report html by adding tables for sources data
    :param sources: Main sources for tables
    :param document: Main HTML document
    :param etable: Reference table
    :param erows: Reference row
    :return:
    """
    for element, value in sources.copy().items():
        titles = titleize(humanize(element))

        if etable and not isinstance(sources[element], list) and not erows:
            with etable:
                trow = tr()
                if not isinstance(value, dict):
                    trow.add(th(titles))
                    trow.add(td(value))
        elif erows:
            with erows:
                if element == value:
                    # Heading
                    erows.add(th(titles))
                else:
                    erows.add(td(value))
        elif not erows and not etable and isinstance(value, str):
            with document:
                with table(id=f'table_{element}', _class="clearfix").add(tbody()) as etable:
                    etable.add(div(_class="pointer"))
                    etable.add(h3(TEXT_GENERAL))  # Heading
                    xrow = tr()
                    xrow.add(th(titles))
                    xrow.add(td(value))

        # If element is dict inspect each key value and clear useless keys
        if isinstance(sources[element], dict):
            with document:
                # div(_class="pointer")
                h3(titles)
                with table(id=f'table_{element}', _class="clearfix").add(tbody()) as tables:
                    builder(sources[element], document, tables)

        # If element is list inspect each key value and clear useless keys
        if isinstance(sources[element], list):
            # How many elements this list has
            size = len(sources[element])

            with document:
                # Make a table
                with table(id=f'table_{element}', _class="clearfix",
                           style="page-break-inside: avoid; margin-top: 2rem;").add(tbody()) as tables:

                    # Add headings
                    with tr(id=f'row_heading_{element}_heading'):
                        columns = 1
                        with suppress(Exception):
                            columns = len(next(iter(sources[element])).keys())

                        with td(colspan=columns) as heading:
                            heading.add(h3(f'{titles} - {size}', _class="clearfix"))

                    # Get each element value
                    for index, elements in enumerate(sources[element]):
                        if isinstance(elements, dict):
                            sources[element][index] = OrderedDict(elements)

                        if index == 0:
                            headings = {}

                            for part in elements.keys():
                                headings.update({part: part})

                            # Add headings
                            with tr(id=f'row_heading_{element}_{index}') as r:
                                builder(headings, document, tables, r)

                        # Add rows
                        with tr(id=f'row_{element}_{index}') as rows:
                            builder(elements, document, tables, rows)

    return sources

def normalize(sources):
    """
    Normalize keys values to strings
    :param sources:
    :return:
    """
    for element, value in sources.copy().items():

        # If element is int convert to string
        if isinstance(sources[element], int):
            sources[element] = str(sources[element])

        # If element is None convert to string
        if sources[element] is None:
            sources[element] = ""

        # If element is dict inspect each key value and clear useless keys
        if isinstance(sources[element], dict):
            normalize(sources[element])

        # If element is list inspect each key value and clear useless keys
        if isinstance(sources[element], list):
            evaluting = []

            for index, elements in enumerate(sources[element]):
                if isinstance(elements, int) or isinstance(elements, str):
                    sources[element][index] = str(elements)
                    evaluting.append(str(elements))

                # Ignored else for clear viewing
                if type(elements) not in [int, str]:
                    normalize(elements)

    return sources

def json_to_html(sources, titled='Report', size=7):
    """
    Convert json to html
    :param sources:
    :param titled:
    :param size:
    :return:
    """

    # Normalizes json fields values
    preparing = normalize(sources)

    # Create HTML Document
    document = dominate.document(title=titled)

    with document.head:
        style(
            'table { page-break-inside:auto } tr { page-break-inside:avoid; page-break-after:auto } thead { display:table-header-group } tfoot { display:table-footer-group } '
            '.pointer { display: block; clear: both; page-break-after: always; } '
            '.clearfix { overflow: auto; clear: both } '
            '.clearfix::after { content: ""; clear: both; display: table; } '
            'body { margin: 0 ' + f'{size}rem;' + ' } '
                                                  'table { width: 100%; font-size: 0.8rem; } '
                                                  'th,td { border: 0.11rem solid black; } '
                                                  'table, th,td { border-collapse: collapse; padding: 0.25rem 1rem; text-align: left; margin: 0; } '
                                                  'tr:nth-child(even) { background-color: #cccccc61; } ')

    builder(preparing, document)

    return preparing, str(document)


from typing import Literal
import torch
from torch import nn
from src.models.natpn.distributions import Posterior

Reduction = Literal["mean", "sum", "none"]

class BayesianLoss(nn.Module):
    """
    The Bayesian loss computes an uncertainty-aware loss based on the parameters of a conjugate
    prior of the target distribution.
    """

    def __init__(self, entropy_weight: float = 0.0, reconst_weight: float = 0.0, reduction: Reduction = "mean"):
        """
        Args:
            entropy_weight: The weight for the entropy regualarizer.
            reconst_weight: The weight for the reconstruction regularizer.
            reduction: The reduction to apply to the loss. Must be one of "mean", "sum", "none".
        """
        super().__init__()
        self.entropy_weight = entropy_weight
        self.reconst_weight = reconst_weight
        self.reduction = reduction
        self.mse = nn.MSELoss(reduction=self.reduction)

    def forward(self, y_pred: Posterior, y_true: torch.Tensor, X: torch.Tensor = None, y_hat: torch.Tensor = None) -> torch.Tensor:
        """
        Computes the loss of the prediction with respect to the target.

        Args:
            y_pred: The posterior distribution predicted by the Natural Posterior Network.
            y_true: The true targets. Either indices for classes of a classification problem or
                the real target values. Must have the same batch shape as ``y_pred``.
            X: The raw input of the network.
            y_hat: The reconstructed input from the decoder.

        Returns:
            The loss, processed according to ``self.reduction``.
        """
        nll = -y_pred.expected_log_likelihood(y_true)
        loss = nll - self.entropy_weight * y_pred.entropy()

        if self.reduction == "mean":
            loss = loss.mean()
            nll = nll.mean()
        elif self.reduction == "sum":
            loss = loss.sum()
            nll = nll.mean()

        # Add the reconstruction loss
        mse = self.mse(X, y_hat) if X != None else 0
        loss = loss + self.reconst_weight * mse

        return loss, nll, mse


import sys
import os
from os import listdir as listdir
from os.path import join
from os.path import isdir
import numpy as np
import dicom2nifti
import mne
import matplotlib.pyplot as plt
sys.path.insert(0, '/imaging/ai05/RED/RED_MEG/resting/analysis/RED_Rest')
from REDTools import sourcespace_setup
from REDTools import sourcespace_command_line
from REDTools import study_info
from REDTools import connectivity
import joblib
#%%

#located in imaging/rs04/RED/DICOMs

# The T1s DICOMS are located in a folder in a folder for each participants
# The name of this folder is the same for each participant
MP_name = 'Series005_CBU_MPRAGE_32chn'
dicoms = '/imaging/rs04/RED/DICOMs'
STRUCTDIR = '/imaging/ai05/RED/RED_MEG/resting/STRUCTURALS'
MAINDIR = '/imaging/ai05/RED/RED_MEG/resting'
folders = [f for f in os.listdir(dicoms) if os.path.isdir(os.path.join(dicoms,f))]
ids = [f.split('_')[0] for f in folders]

#get ids etc
RED_id, MR_id, MEG_fname = study_info.get_info()
#change fsaverage so they can find

#%% define function for envelope corellations
# input settings for this function and then run
i = 4
invdir = join(MAINDIR, 'invs')
rawdir = join(MAINDIR, 'preprocessed')
outdir = join(MAINDIR, 'envelope_cors')
fsdir = join(MAINDIR, 'STRUCTURALS','FS_SUBDIR')
MEG_fname = MEG_fname
RED_id = RED_id
MR_id = MR_id
method = 'MNE'
combine = 'mean'
lambda2 = 1. / 9.
pick_ori = 'normal'
parc = 'aparc'
time_course_mode = 'mean'

#frequencies we are gonna do
freqs = {'Theta_s':(4,7),
         'Alpha_s':(8,12),
         'Lower Beta_s': (13, 20),
         'Upper Beta_s':(21, 30)}

# make list of dicts for input (we're gonna use job lib for this)
dictlist = []
for f in freqs:
    freq = freqs[f]
    freqname = f
    dictlist.append(dict(
        i = i,
        invdir = invdir,
        rawdir = rawdir,
        outdir = outdir,
        fsdir = fsdir,
        MEG_fname = MEG_fname,
        RED_id = RED_id,
        MR_id = MR_id,
        method = method,
        combine = combine,
        lambda2 =lambda2,
        pick_ori = pick_ori,
        parc = parc,
        time_course_mode = time_course_mode,
        freq = freq,
        freqname = freqname,
    ))

joblib.Parallel(n_jobs=len(dictlist))(
    joblib.delayed(connectivity.envelope_corellation)(indict) for indict in dictlist)

#%% or instead use cluster version
connectivity.cluster_envelope_corellation(MR_id, MAINDIR)

#%% Calculate average corellation matrix for each brain
aec_dir = join(MAINDIR, 'envelope_cors')
thetas = [f for f in listdir(aec_dir) if 'Theta' in f] # find thetas
thetas = [thetas[[i for i, s in enumerate(thetas) if i_d in s][0]] for i_d in RED_id] # reorder to match RED
#repeat for all bands
alphas =[f for f in listdir(aec_dir) if 'Alpha' in f]
alphas = [alphas[[i for i, s in enumerate(alphas) if i_d in s][0]] for i_d in RED_id]
betalow = [f for f in listdir(aec_dir) if 'Lower Beta' in f]
betalow = [betalow[[i for i, s in enumerate(betalow) if i_d in s][0]] for i_d in RED_id]
betahigh =[f for f in listdir(aec_dir) if 'Upper Beta' in f]
betahigh = [betahigh[[i for i, s in enumerate(betahigh) if i_d in s][0]] for i_d in RED_id]

# Combine each band into an average connectivity matrix
arraylist = []
for band in [thetas, alphas, betalow, betahigh]:
    #intialise empty array
    mat_3d = np.zeros(shape=(68, 68, len(band)))
    for i, f in enumerate(band):
        mat_3d[:,:,i] = np.load(join(aec_dir, f))
    arraylist.append(mat_3d)

#%% make a basic average array for one frequency
freq_labels = ['thetas', 'alphas', 'betalow', 'betahigh']
threshold = 0.05 # threshold for strength
perc_part = 0.70 # Threshold for how many participants
for i in range(len(freq_labels)):
    thisarray = arraylist[i]
    corr = thisarray.mean(axis=2) # simple mean
    # try thresholding per participants
    bools = thisarray > threshold # where we are above threshold
    chn_mask = np.zeros(thisarray.shape[0:2], dtype=bool)
    corr_masked = np.zeros(thisarray.shape[0:2])
    for ii in range(thisarray.shape[0]): #channels where perce participants above threshold
        for iii in range(thisarray.shape[1]):
            perc = sum(bools[ii,iii])/len(bools[ii,iii])
            print(perc)
            if perc < perc_part:
                chn_mask[ii,iii] = False
                corr_masked[ii,iii] = 0
            else:
                chn_mask[ii,iii] = True
                corr_masked[ii,iii] = corr[ii,iii]
    corr2plot = corr_masked
    nlines=300

    #% plot corellation matrix
    # let's plot this matrix
    fig, ax = plt.subplots(figsize=(4, 4))
    ax.imshow(corr2plot, cmap='viridis', clim=np.percentile(corr, [5, 95]))
    fig.tight_layout()
    plt.savefig(f'/imaging/ai05/images/AEC{freq_labels[i]}_group_mat.png')

    #% plot this on a brain
    threshold_prop = 0.3  # percentage of strongest edges to keep in the graph
    #degree = mne.connectivity.degree(corr, threshold_prop=threshold_prop) # get the degree
    degree = mne.connectivity.degree(corr2plot) # get the degree
    #load in FSAVERAGE for parcellations and inverse for plotting
    labels = mne.read_labels_from_annot('fsaverage_1', parc='aparc',
                                        subjects_dir=join(MAINDIR, 'STRUCTURALS','FS_SUBDIR'))
    labels = labels[0:-1]
    # we know that 99144's inversion contains source from FSAVERAGE so load that
    inv_op = mne.minimum_norm.read_inverse_operator(join(invdir, '99144-inv.fif'))
    stc = mne.labels_to_stc(labels, degree)
    stc = stc.in_label(mne.Label(inv_op['src'][0]['vertno'], hemi='lh') +
                       mne.Label(inv_op['src'][1]['vertno'], hemi='rh'))
    stc.plot(
        clim=dict(kind='percent', lims=[75, 85, 95]), colormap='gnuplot',
        subjects_dir=join(STRUCTDIR, 'FS_SUBDIR'), hemi='lh',
        smoothing_steps=1, time_label=freq_labels[i], backend='matplotlib').savefig(f'/imaging/ai05/images/AEC_{freq_labels[i]}_src_group_lh.png')
    stc.plot(
        clim=dict(kind='percent', lims=[75, 85, 95]), colormap='gnuplot',
        subjects_dir=join(STRUCTDIR, 'FS_SUBDIR'), hemi='rh',
        smoothing_steps=1, time_label=freq_labels[i], backend='matplotlib').savefig(f'/imaging/ai05/images/AEC_{freq_labels[i]}_src_group_rh.png')

    #% make a circle plot
    # First, we reorder the labels based on their location in the left hemi
    label_names = [label.name for label in labels]
    lh_labels = [name for name in label_names if name.endswith('lh')]

    # Get the y-location of the label
    label_ypos = list()
    for name in lh_labels:
        idx = label_names.index(name)
        ypos = np.mean(labels[idx].pos[:, 1])
        label_ypos.append(ypos)

    # Reorder the labels based on their location
    lh_labels = [label for (yp, label) in sorted(zip(label_ypos, lh_labels))]

    # For the right hemi
    rh_labels = [label[:-2] + 'rh' for label in lh_labels]

    # Save the plot order and create a circular layout
    node_order = list()
    node_order.extend(lh_labels[::-1])  # reverse the order
    node_order.extend(rh_labels)

    node_angles = mne.viz.circular_layout(label_names, node_order, start_pos=90,
                                  group_boundaries=[0, len(label_names) / 2])
    label_colors = [label.color for label in labels]
    # Plot the graph using node colors from the FreeSurfer parcellation. We only
    # show the 300 strongest connections.
    mne.viz.plot_connectivity_circle(corr2plot, label_names, n_lines=nlines,
                             node_angles=node_angles, node_colors=label_colors,
                             title=f'{freq_labels[i]}')[0].savefig(f'/imaging/ai05/images/AEC_{freq_labels[i]}_circle.png')

    plotting.plot_connectome(corr_masked, coords,
                             edge_threshold="95%",
                             title=f'{freq_labels[i]}').savefig(f'/imaging/ai05/images/AEC_{freq_labels[i]}net.png')
#%% vertex to MNI space
coords = []
for i in range(len(labels)):
    if 'lh' in label_names[i]:
        hem = 1
    else:
        hem = 0
    coord = mne.vertex_to_mni(labels[i].center_of_mass(subjects_dir=fsdir), subject='fsaverage_1',hemis=hem,subjects_dir=fsdir)
    coords.append(coord[0])
#%% nii learn plots
from nilearn import datasets
from nilearn.input_data import NiftiLabelsMasker
from nilearn import plotting
des = datasets.fetch_atlas_surf_destrieux()
# create masker to extract functional data within atlas parcels
masker = NiftiLabelsMasker(labels_img=labels, standardize=True,
                           memory='nilearn_cache')
coordinates = plotting.find_parcellation_cut_coords(labels_img='/imaging/ai05/RED/RED_MEG/resting/analysis/aparcaseg.nii.gz')

#%% Meeting notes from 6/2/2019
#Continous concetantion, downsample after hilbert envelope
#kanads method section of PhD -- threshold free level setting (singular value decomposition)
#

#%% source recon continous raw data
pythonpath = '/home/ai05/.conda/envs/mne_2/bin/python'
for i in range(len(RED_id)):
    pycom = f"""
import sys
import os
from os.path import join
import numpy as np
import mne
sys.path.insert(0, '/imaging/ai05/RED/RED_MEG/resting/analysis/RED_Rest')
from REDTools import study_info

MP_name = 'Series005_CBU_MPRAGE_32chn'
dicoms = '/imaging/rs04/RED/DICOMs'
STRUCTDIR = '/imaging/ai05/RED/RED_MEG/resting/STRUCTURALS'
MAINDIR = '/imaging/ai05/RED/RED_MEG/resting'
folders = [f for f in os.listdir(dicoms) if os.path.isdir(os.path.join(dicoms,f))]
ids = [f.split('_')[0] for f in folders]

#get ids etc
RED_id, MR_id, MEG_fname = study_info.get_info()

def raw2source(meg_f, inv_op, method, outdir):
    raw = mne.io.read_raw_fif(meg_f, preload=True)
    inv = mne.minimum_norm.read_inverse_operator(inv_op)
    start, stop = raw.time_as_index([60, raw.times[-30000]])
    stc = mne.minimum_norm.apply_inverse_raw(raw, inv,
                                             lambda2=1.0/1.0**2,
                                             method=method,
                                             start=start,
                                             stop=stop,
                                             buffer_size=int(len(raw.times)/10))
    np.save(join(outdir, f'{{os.path.basename(meg_f).split("_")[0]}}_{{method}}_stc_data.npy'),stc.data)
    np.save(join(outdir, f'{{os.path.basename(meg_f).split("_")[0]}}_{{method}}_stc_vertices.npy'),stc.vertices)
    np.save(join(outdir, f'{{os.path.basename(meg_f).split("_")[0]}}_{{method}}_stc_times.npy'),stc.times)

invdir = join(MAINDIR, 'inverse_ops')
rawdir = join(MAINDIR, 'preprocessed')
fsdir = join(MAINDIR, 'STRUCTURALS','FS_SUBDIR')
meg_f = join(rawdir, MEG_fname[{i}])
inv_op = join(invdir, f'{{RED_id[{i}]}}-inv.fif')
outdir = join(MAINDIR, 'raw_stcs')
method = 'MNE'
raw2source(meg_f, inv_op, method,outdir)
    """

    # save to file
    print(pycom, file=open(join(MAINDIR, 'cluster_scripts', f'{i}_aec.py'), 'w'))

    # construct csh file
    tcshf = f"""#!/bin/tcsh
            {pythonpath} {join(MAINDIR, 'cluster_scripts', f'{i}_aec.py')}
                    """
    # save to directory
    print(tcshf, file=open(join(MAINDIR, 'cluster_scripts', f'{i}_aec.csh'), 'w'))

    # execute this on the cluster
    os.system(f"sbatch --job-name=AmpEnvConnect_{i} --mincpus=2 --mem-per-cpu=40G -t 0-1:00 {join(MAINDIR, 'cluster_scripts', f'{i}_aec.csh')}")

#%% Read in a file
from scipy.signal import hilbert, resample
import scipy as ss
i = 10
outdir = join(MAINDIR, 'raw_stcs')
method = 'MNE'
data = np.load(join(outdir, f'{os.path.basename(MEG_fname[i]).split("_")[0]}_{method}_stc_data.npy'))
#%% Calculate the hilbert envelope for all the voxels and save downsampled version
downsample_factor = 100
#analytic_signal = np.zeros((data.shape[0], int(data.shape[1]/downsample_factor)))
analytic_signal = np.zeros(data.shape)
for i in range(data.shape[0]):
    tmp_sig = hilbert(data[i,:])
    #analytic_signal[i,:] = resample(tmp_sig, int(data.shape[1]/downsample_factor))
    analytic_signal[i,:] = tmp_sig
    print(i/data.shape[0])

#%% downsample
ii = 1
#plot some stuff
fig, ax = plt.subplots(1)

section = data[ii,0:4000]

section = mne.filter.filter_data(section,1000,freqs['Alpha'][0], freqs['Alpha'][1])
#get number of fft
n_fft = mne.filter.next_fast_len(len(section))
env = hilbert(section,N=n_fft, axis =-1)
d_env = resample(np.abs(env),int(len(env)/25))
d_sec = resample(section,int(len(section)/25))
#ax.plot(section[1000:5000])
#ax.plot(np.abs(env)[1000:5000])
#ax.plot(section)
ax.plot(d_sec)
ax.plot(d_env)
fig.savefig('/home/ai05/test.png')

#%% convert example into MNI space
i = 0

def mni_conv(i):
    inv_op = mne.minimum_norm.read_inverse_operator(join(invdir, f'{RED_id[i]}-inv.fif'))
    # get raw file
    raw = mne.io.read_raw_fif(join(rawdir, MEG_fname[i]), preload=False)
    events = mne.make_fixed_length_events(raw, duration=10.)
    epochs = mne.Epochs(raw, events=events, tmin=0, tmax=10.,
                        baseline=None, preload=True)
    del raw
    epochs = epochs[0:1]
    # invert the eopchs
    stcs = mne.minimum_norm.apply_inverse_epochs(epochs, inv_op,
                                                 lambda2=lambda2,
                                                 pick_ori=pick_ori,
                                                 method=method)
    stc = stcs[0]
    del stcs
    mni = mne.vertex_to_mni(stc.vertices, [0,1], MR_id[i], subjects_dir=fsdir) # mni space
    mni = np.vstack([mni[0], mni[1]]) # stack
    outdir = join(MAINDIR, 'mni_coords')
    np.save(join(outdir, f'{os.path.basename(MEG_fname[i]).split("_")[0]}_mni.npy'),mni)

joblib.Parallel(n_jobs=15)(
    joblib.delayed(mni_conv)(i) for i in range(len(MEG_fname)))


import os,sys, json
import time,math
import logging
logger = logging.getLogger('luigi-interface')
alloc_log = logging.getLogger('alloc_log')
alloc_log.setLevel(logging.DEBUG)

import luigi
from luigi.contrib.slurm import SlurmExecutableTask
from luigi.contrib.scattergather import ScatterGather
from luigi.util import requires, inherits
from luigi import LocalTarget
from luigi.file import TemporaryFile

from src.utils import CheckTargetNonEmpty
from src.SGUtils import ScatterBED, GatherVCF, ScatterVCF

picard="java -XX:+UseSerialGC -Xmx{mem}M -jar /tgac/software/testing/picardtools/2.1.1/x86_64/bin/picard.jar"
gatk="java -XX:+UseSerialGC -Xmx{mem}M -jar /tgac/software/testing/gatk/3.6.0/x86_64/bin/GenomeAnalysisTK.jar "
python="source /usr/users/ga004/buntingd/FP_dev/dev/bin/activate"

# Ugly hack
script_dir = os.path.join(os.path.split(os.path.split(__file__)[0])[0], 'scripts')
log_dir = os.path.join(os.path.split(os.path.split(os.path.split(__file__)[0])[0])[0], 'logs')
os.makedirs(log_dir, exist_ok=True)

'''
Guidelines for harmonious living:
--------------------------------
1. Tasks acting on fastq files should output() a list like [_R1.fastq, _R2.fastq]
2. Tasks acting on a bam should just output() a single Target
3. Tasks acting on a vcf should just output() a single Target

Notes
-------
job.mem is actually mem_per_cpu
'''

N_scatter = int(sys.argv[2]) if __name__ == '__main__' else 5

class GenomeContigs(luigi.ExternalTask):
    '''one per line list of contigs in the genome'''
    mask = luigi.Parameter()
    def output(self):
        return LocalTarget(self.mask)

@ScatterGather(ScatterBED, GatherVCF, N_scatter)
@inherits(GenomeContigs)
class GenotypeGVCF(SlurmExecutableTask, CheckTargetNonEmpty):
    '''Combine the per sample g.vcfs into a complete callset
    :param str output_prefix: '''
    output_prefix = luigi.Parameter()
    base_dir = luigi.Parameter(significant=False)
    scratch_dir = luigi.Parameter(default="/tgac/scratch/buntingd/", significant=False)
    reference = luigi.Parameter()
    lib_list = luigi.ListParameter()

    def requires(self):
        return self.clone(GenomeContigs)

    def __init__(self, *args, **kwargs):
        super().__init__(*args, **kwargs)
        # Set the SLURM request params for this task
        self.mem = 8000
        self.n_cpu = 1
        self.partition = "tgac-medium"
        self.variants = [os.path.join(self.base_dir, 'libraries', library, library + ".g.vcf") for library in self.lib_list]

    def output(self):
        return LocalTarget(os.path.join(self.base_dir, 'callsets', self.output_prefix, self.output_prefix+"_raw.vcf.gz"))

    def work_script(self):
        return '''#!/bin/bash
                source jre-8u92
                source gatk-3.6.0
                gatk='{gatk}'

                set -eo pipefail
                $gatk -T GenotypeGVCFs -R {reference} -L {intervals} -o {output}.temp.vcf.gz --includeNonVariantSites {variants}

                mv {output}.temp.vcf.gz {output}
                '''.format(output=self.output().path,
                           intervals=self.input().path,
                           gatk=gatk.format(mem=self.mem*self.n_cpu),
                           reference=self.reference,
                           variants="\\\n".join([" --variant "+ lib for lib in self.variants]) )

@ScatterGather(ScatterVCF, GatherVCF, N_scatter)
@inherits(GenotypeGVCF)
class VcfToolsFilter(SlurmExecutableTask, CheckTargetNonEmpty):
    '''Applies hard filtering to the raw callset'''
    GQ = luigi.IntParameter(default=30)
    QD = luigi.IntParameter(default=5)
    FS = luigi.IntParameter(default=30)

    def requires(self):
        return self.clone(GenotypeGVCF)

    def __init__(self, *args, **kwargs):
        super().__init__(*args, **kwargs)
        # Set the SLURM request params for this task
        self.mem = 8000
        self.n_cpu = 2
        self.partition = "tgac-medium"

    def output(self):
        return LocalTarget(os.path.join(self.base_dir, 'callsets', self.output_prefix , self.output_prefix + "_filtered.vcf.gz"))

    def work_script(self):
        self.temp1 = TemporaryFile()
        self.temp2 = TemporaryFile()

        return '''#!/bin/bash
                source vcftools-0.1.13;
                source bcftools-1.3.1;
                set -eo pipefail

                bcftools view --apply-filters . {input} -o {temp1} -O z --threads 1
                bcftools filter {temp1} -e "FMT/RGQ < {GQ} || FMT/GQ < {GQ} || QD < {QD} || FS > {FS}" --set-GTs . -o {temp2} -O z --threads 1
                vcftools --gzvcf {temp2} --recode --max-missing 0.000001 --stdout --bed {mask} | bgzip -c > {output}.temp

                mv {output}.temp {output}
                tabix -p vcf {output}
                '''.format(input=self.input().path,
                           output=self.output().path,
                           GQ=self.GQ,
                           QD=self.QD,
                           FS=self.FS,
                           mask=self.mask,
                           temp1=self.temp1.path,
                           temp2=self.temp2.path)

@ScatterGather(ScatterVCF, GatherVCF, N_scatter)
@inherits(VcfToolsFilter)
class GetSNPs(SlurmExecutableTask, CheckTargetNonEmpty):
    '''Extracts just sites with only biallelic SNPs that have a least one variant isolate'''
    def requires(self):
        return self.clone(VcfToolsFilter)

    def __init__(self, *args, **kwargs):
        super().__init__(*args, **kwargs)
        # Set the SLURM request params for this task
        self.mem = 4000
        self.n_cpu = 1
        self.partition = "tgac-medium"

    def output(self):
        return LocalTarget(os.path.join(self.base_dir, 'callsets', self.output_prefix , self.output_prefix + "_SNPs.vcf.gz"))

    def work_script(self):
        return '''#!/bin/bash
                  source jre-8u92
                  source gatk-3.6.0
                  gatk='{gatk}'
                  set -eo pipefail

                  $gatk -T -T SelectVariants -V {input} -R {reference} --restrictAllelesTo BIALLELIC --selectTypeToInclude SNP --out {output}.temp.vcf.gz

                  mv {output}.temp.vcf.gz {output}
                  '''.format(input=self.input().path,
                             output=self.output().path,
                             reference=self.reference,
                             gatk=gatk.format(mem=self.mem*self.n_cpu))

@ScatterGather(ScatterVCF, GatherVCF, N_scatter)
@inherits(VcfToolsFilter)
class GetINDELs(SlurmExecutableTask, CheckTargetNonEmpty):
    '''Get sites with MNPs'''
    def requires(self):
        return self.clone(VcfToolsFilter)

    def __init__(self, *args, **kwargs):
        super().__init__(*args, **kwargs)
        # Set the SLURM request params for this task
        self.mem = 4000
        self.n_cpu = 1
        self.partition = "tgac-medium"

    def output(self):
        return LocalTarget(os.path.join(self.base_dir, 'callsets', self.output_prefix , self.output_prefix + "_INDELs_only.vcf.gz"))

    def work_script(self):
        return '''#!/bin/bash
                  source jre-8u92
                  source gatk-3.6.0
                  gatk='{gatk}'
                  set -eo pipefail

                  $gatk -T -T SelectVariants -V {input} -R {reference} --selectTypeToInclude MNP  --selectTypeToInclude MIXED   --out {output}.temp.vcf.gz

                  mv {output}.temp.vcf.gz {output}
                  '''.format(input=self.input().path,
                             output=self.output().path,
                             reference=self.reference,
                             gatk=gatk.format(mem=self.mem*self.n_cpu))

@ScatterGather(ScatterVCF, GatherVCF, N_scatter)
@inherits(VcfToolsFilter)
class GetRefSNPs(SlurmExecutableTask, CheckTargetNonEmpty):
    '''Create a VCF with SNPs and include sites that are reference like in all samples'''
    def requires(self):
        return self.clone(VcfToolsFilter)
    def __init__(self, *args, **kwargs):
        super().__init__(*args, **kwargs)
        # Set the SLURM request params for this task
        self.mem = 4000
        self.n_cpu = 1
        self.partition = "tgac-medium"

    def output(self):
        return LocalTarget(os.path.join(self.base_dir, 'callsets', self.output_prefix , self.output_prefix + "_RefSNPs.vcf.gz"))

    def work_script(self):
        return '''#!/bin/bash
                  source jre-8u92
                  source gatk-3.6.0
                  gatk='{gatk}'
                  set -eo pipefail

                  $gatk -T -T SelectVariants -V {input} -R {reference} --restrictAllelesTo BIALLELIC --selectTypeToInclude SYMBOLIC --selectTypeToInclude NO_VARIATION  --selectTypeToInclude SNP  --out {output}.temp.vcf.gz

                  mv {output}.temp.vcf.gz {output}
                  '''.format(input=self.input().path,
                             output=self.output().path,
                             reference=self.reference,
                             gatk=gatk.format(mem=self.mem*self.n_cpu))

#-----------------------------------------------------------------------#

@inherits(GetSNPs)
@inherits(GetRefSNPs)
@inherits(GetINDELs)
class CallsetWrapper(luigi.WrapperTask):
    def requires(self):
        yield self.clone(GetINDELs)
        yield self.clone(GetSNPs)
        yield self.clone(GetRefSNPs)

if __name__ == '__main__':
    os.environ['TMPDIR'] = "/tgac/scratch/buntingd"
    logging.disable(logging.DEBUG)
    timestr = time.strftime("%Y%m%d-%H%M%S")

    fh = logging.FileHandler(os.path.join(log_dir, os.path.basename(__file__) + "_" + timestr + ".log"))
    fh.setLevel(logging.INFO)
    formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
    fh.setFormatter(formatter)
    logger.addHandler(fh)

    alloc_fh = logging.FileHandler(os.path.join(log_dir, os.path.basename(__file__) + "_" + timestr + ".salloc.log"))
    alloc_fh.setLevel(logging.INFO)
    formatter = logging.Formatter('%(message)s')
    alloc_fh.setFormatter(formatter)
    alloc_log.addHandler(alloc_fh)

    with open(sys.argv[1], 'r') as libs_file:
        lib_list = [line.rstrip() for line in libs_file]

    name = os.path.split(sys.argv[1])[1].split('.', 1)[0]

    luigi.run(['CallsetWrapper', '--output-prefix', name,
                                 '--lib-list', json.dumps(lib_list),
                                 '--reference', '/tgac/workarea/collaborators/saunderslab/Realignment/data/PST130_contigs.fasta',
                                 '--mask', '/tgac/workarea/users/buntingd/realignment/PST130/Combined/PST130_RNASeq_collapsed_exons.bed'] + sys.argv[3:])


import math

from ultralytics import YOLO
import cv2
import cvzone
from sort import *

capture = cv2.VideoCapture("../../workshopProject/Videos/people.mp4")

model = YOLO("../Yolo-Weights/yolov8l.pt")

classNames = ["person", "bicycle", "car", "motorbike", "aeroplane", "bus", "train", "truck", "boat",
              "traffic light", "fire hydrant", "stop sign", "parking meter", "bench", "bird", "cat",
              "dog", "horse", "sheep", "cow", "elephant", "bear", "zebra", "giraffe", "backpack", "umbrella",
              "handbag", "tie", "suitcase", "frisbee", "skis", "snowboard", "sports ball", "kite", "baseball bat",
              "baseball glove", "skateboard", "surfboard", "tennis racket", "bottle", "wine glass", "cup",
              "fork", "knife", "spoon", "bowl", "banana", "apple", "sandwich", "orange", "broccoli",
              "carrot", "hot dog", "pizza", "donut", "cake", "chair", "sofa", "pottedplant", "bed",
              "diningtable", "toilet", "tvmonitor", "laptop", "mouse", "remote", "keyboard", "cell phone",
              "microwave", "oven", "toaster", "sink", "refrigerator", "book", "clock", "vase", "scissors",
              "teddy bear", "hair drier", "toothbrush"
              ]

mask = cv2.imread("mask2.png")

tracker = Sort(max_age=20, min_hits=3, iou_threshold=0.3)

limitsUp = [103, 161, 296, 161]
limitsDown = [527, 489, 735, 489]

counterUp = []
counterDown = []

while True:
    success, img = capture.read()
    detectRegion = cv2.bitwise_and(img, mask)
    results = model(detectRegion, stream=True)
    detections = np.empty((0, 5))

    for r in results:
        boxes = r.boxes
        for box in boxes:
            # Bounding box
            x1, y1, x2, y2 = box.xyxy[0]
            x1, y1, x2, y2 = int(x1), int(y1), int(x2), int(y2)
            w, h = x2 - x1, y2 - y1
            boundingbox = int(x1), int(y1), int(w), int(h)

            confidence_value = math.ceil((box.conf[0] * 100)) / 100

            cls = int(box.cls[0])

            currentClass = classNames[cls]
            if currentClass == 'person' and confidence_value > 0.3:
                cvzone.putTextRect(img, f'{classNames[cls]} {confidence_value}', (max(0, x1), max(40, y1)),
                                   scale=0.7, thickness=1, offset=3)

                currentArray = np.array([x1, y1, x2, y2, confidence_value])

                detections = np.vstack((detections, currentArray))

    resultsTracker = tracker.update(detections)

    cv2.line(img, (limitsUp[0], limitsUp[1]), (limitsUp[2], limitsUp[3]), (0, 0, 255), 5)
    cv2.line(img, (limitsDown[0], limitsDown[1]), (limitsDown[2], limitsDown[3]), (0, 0, 255), 5)

    for result in resultsTracker:
        x1, y1, x2, y2, id = result
        x1, y1, x2, y2 = int(x1), int(y1), int(x2), int(y2)
        print(result)
        w, h = x2 - x1, y2 - y1
        boundingbox = int(x1), int(y1), int(w), int(h)

        cvzone.cornerRect(img, boundingbox, l=5, rt=2, colorR=(255, 0, 0))
        cvzone.putTextRect(img, f'{int(id)}', (max(0, x1), max(40, y1)),
                           scale=1, thickness=1, offset=5)

        cx, cy = x1 + w // 2, y1 + h // 2
        cv2.circle(img, (cx, cy), 5, (255, 0, 255), cv2.FILLED)

        if limitsUp[0] < cx < limitsUp[2] and limitsUp[1] - 15 < cy < limitsUp[1] + 15:
            if counterUp.count(id) == 0:
                counterUp.append(id)
                cv2.line(img, (limitsUp[0], limitsUp[1]), (limitsUp[2], limitsUp[3]), (0, 255, 0), 5)

        if limitsDown[0] < cx < limitsDown[2] and limitsDown[1] - 15 < cy < limitsDown[1] + 15:
            if counterDown.count(id) == 0:
                counterDown.append(id)
                cv2.line(img, (limitsDown[0], limitsDown[1]), (limitsDown[2], limitsDown[3]), (0, 255, 0), 5)

    cvzone.putTextRect(img, f'Up: {len(counterUp)}', (929, 345))
    cvzone.putTextRect(img, f'Down: {len(counterDown)}', (929, 395))

    cv2.imshow("Image", img)
    # cv2.imshow("DetectRegion", detectRegion)
    cv2.waitKey(0)

object-detection/Project 2 - People Counter/People-Counter.py


import openpyxl
wb = openpyxl.load_workbook('main.xlsx')

sheet = wb['USER']

def add_score(u_id,score):
    cell = sheet['F'+str(u_id)]
    old_score = cell.value
    cell.value = old_score + score
    wb.save('main.xlsx')


from argparse import ArgumentParser
from typing import List, Optional
from pathlib import Path

from config import EvalConfig

class _ArgBuilder:
    def __init__(self, name: Optional[str] = None):
        if name is None:
            self._argparser = ArgumentParser()
        else:
            self._argparser = ArgumentParser(name)

    def _add_unary_arg(self, *args, **kwargs) -> None:
        self._argparser.add_argument(
            *args, nargs='?', action='store', **kwargs)

    def build(self):
        return self._argparser

class _EvalArgBuilder(_ArgBuilder):
    def __init__(self):
        super().__init__(name='pyeval')

        self._add_unary_arg('--executable', type=str, required=True)
        self._add_unary_arg('--args', type=str, required=True)
        self._add_unary_arg('--threads', type=int, default=1)
        self._add_unary_arg('--database', type=str, default='pyeval.db')
        self._add_unary_arg('--work-dir', type=str, default='/tmp/pyeval')
        self._add_unary_arg('--reference', type=str, required=True)
        self._add_unary_arg('--comment', type=str)

class _RaconArgBuilder(_ArgBuilder):
    def __init__(self):
        super().__init__('correction')
        self._add_unary_arg('--window-length', type=int, default=500)
        self._add_unary_arg('--error-threshold', type=float, default=0.3)
        self._argparser.add_argument('reads')
        self._argparser.add_argument('overlaps')

class _CamelArgBuilder(_RaconArgBuilder):
    def __init__(self):
        super().__init__()

        self._add_unary_arg('--n-overlaps', type=int, default=128)

def parse_args(args: Optional[List[str]] = None) -> EvalConfig:
    eval_args_parser = _EvalArgBuilder().build()

    if args:
        print(args)
        eval_args = eval_args_parser.parse_args(args)
    else:
        eval_args = eval_args_parser.parse_args()

    if eval_args.executable == 'camel':
        exe_args_parser = _CamelArgBuilder().build()
    elif eval_args.executable == 'racon':
        exe_args_parser = _RaconArgBuilder().build()
    else:
        raise RuntimeError('[pyeval] unsupported')

    exe_args = {'threads': eval_args.threads}
    parsed_args = vars(exe_args_parser.parse_args(eval_args.args.split(' ')))
    for k, v in parsed_args.items():
        exe_args[f'{k.replace("_", "-")}'] = v

    return EvalConfig(
        Path(eval_args.work_dir),
        eval_args.executable,
        exe_args,
        eval_args.threads,
        Path(eval_args.reference),
        Path(eval_args.database),
        eval_args.comment)


import scrapy

from lp_scraper.items import ImageItem, MushroomItem

class LPSpider(scrapy.Spider):
  name = 'lp_spider'
  start_urls = ['http://www.luontoportti.com/suomi/fi/sienet/akansieni']

  def parse(self, response):
    name_fin = response.css('div#teksti h3 ::text').extract_first()
    name_latin = response.css('div#teksti h4 ::text').extract_first()
    # Used for naming the pictures eg. "Chlorophyllum rhacodes" -> "chlorophyllum_rhacodes"
    latin_formatted = name_latin.lower().replace(' ', '_')

    for i, li_item in enumerate(response.css('div#teksti ul li')):
      if i == 2:
        # Third item is edibility eg. 'Syötävyys: O** – keittämisen jälkeen hyvä ruokasieni'
        edibility = li_item.css('::text').extract_first().lower().strip()

    for i, img_link in enumerate(response.css('div#kuvat a')):
      link_url = response.urljoin(img_link.css('::attr(href)').extract_first().strip())
      # Picks last 4 characters from the link -> 1.jpg / .jpeg / 1.png
      # And then splits them by "." and takes the last item from the list
      img_ext = link_url[-4:].split('.')[-1].lower()
      name_img = "{}{}.{}".format(latin_formatted, i, img_ext)
      img_dict = {
        'name_latin': name_latin,
        'name_fin': name_fin,
        'edibility': edibility,
        'name_img': name_img,
        'img_url': link_url,
      }
      yield ImageItem(img_dict)

    # List of links, only 1 in the first and the last page (of all mushrooms), otherwise 2
    nav_links = response.css('div.float_l a[rel=prev]')
    next_url = nav_links[len(nav_links) - 1].css('::attr(href)').extract_first()
    shroom_dict = {
      'name_latin': name_latin,
      'name_fin': name_fin,
      'edibility': edibility,
      'url_lp': response.url
    }
    yield MushroomItem(shroom_dict)
    yield response.follow(next_url)


import h5py
import pandas as pd
import matplotlib.pyplot as plt
import numpy as np
from auxiliary import get_perifery
from plot import simulation_clone_history as smh
from scipy import stats

import matplotlib
plt.rcParams.update({'font.size': 22,
                    'font.weight': 'normal'})
plt.style.use('dark_background')
def get_counts_structure(files):
    file_index = 1;

    count_df = pd.DataFrame()
    for filename in files:
        #filename = 'D:\Serhii\Projects\EvoChan\Simulations\sim_results_garching\\20210426_large_statistics\\23_04_run_'+str(q)+'\output\\all_data.h5';
        #filename = 'D:\Serhii\Projects\EvoChan\Simulations\sim_results_garching\\20210517_switch_control\\17_05_switch_at_'+str(q)+'0_1\output\\all_data.h5';
        TitleName = 'run3';
        f = h5py.File(filename, 'r')

        history_read = pd.read_hdf(filename,'data/history');
        history_read = history_read.fillna(value=0.)
        #print(history_read)
        color_history_read = pd.read_hdf(filename,'data/color_history');
        color_mask_type_1 = color_history_read.copy();
        color_mask_type_1[color_mask_type_1 == 2.] = 0.
        color_mask_type_1 = color_mask_type_1.fillna(value = 0)

        color_mask_type_2 = color_history_read.copy();
        color_mask_type_2[color_mask_type_2 == 1.] = 0.
        color_mask_type_2[color_mask_type_2 == 2.] = 1.
        color_mask_type_2 = color_mask_type_2.fillna(value = 0)
        #print(color_mask_type_1)
        #print(color_history_read)

        tp1 = history_read * color_mask_type_1.values
        tp2 = history_read * color_mask_type_2.values

        tp1_array = np.array(tp1)
        #print(tp1)
        df = pd.DataFrame()
        for i in range(len(tp1_array)):
            (unique, counts) = np.unique(tp1_array[i], return_counts = True)
            #print(unique)
            unique_str = [str(unique[i]) for i in range(len(unique))]
            #print(unique_str)
            dictionary = {unique_str[i] : [counts[i]] for i in range(len(counts))}
        # print(dictionary)
            data = pd.DataFrame.from_dict(dictionary)

            #print(data)
            df=pd.concat([df,data],axis = 0, ignore_index = True)

        #df = df.fillna(value = 0.)

        tp2_array = np.array(tp2)
        #print(tp2)
        df2 = pd.DataFrame()
        for i in range(len(tp2_array)):
            (unique, counts) = np.unique(tp2_array[i], return_counts = True)
            #print(unique)
            unique_str = [str(unique[i]) for i in range(len(unique))]
            #print(unique_str)
            dictionary = {unique_str[i] : [counts[i]] for i in range(len(counts))}
        # print(dictionary)
            data = pd.DataFrame.from_dict(dictionary)

            #print(data)
            df2=pd.concat([df2,data],axis = 0, ignore_index = True)

        #sorting

        lengths2 = []
        keys2 = []
        for key in df2.keys():
            x = df2.index
            y = float(key)*np.ones(len(x))
            width = df2[key].values
            length = np.sum(~np.isnan(width))
            lengths2.append(length)
            keys2.append(float(key))

        df_sort2 = pd.DataFrame(lengths2)
        df_sort2 = df_sort2.T
        df_sort2.columns = keys2

        lengths = []
        keys = []
        for key in df.keys():
            x = df.index
            y = float(key)*np.ones(len(x))
            width = df[key].values
            length = np.sum(~np.isnan(width))
            lengths.append(length)
            keys.append(float(key))

        df_sort = pd.DataFrame(lengths)
        df_sort = df_sort.T
        df_sort.columns = keys

        combined_df = df_sort.add(df_sort2, fill_value=0)
        combined_df = combined_df.sort_values(by=0, axis = 1)
        #plt.ylim([3*7500,3*7800])

        countMatrix = np.array(history_read.values)

        tot = []
        for i in range(len(countMatrix)):
            (unique, counts) = np.unique(countMatrix[i], return_counts = True)

            tot_num = len(unique)-1
            tot.append(tot_num)

        interm_df = pd.DataFrame(tot, columns = [str(file_index)]);
        file_index+=1;
        count_df = pd.concat([count_df, interm_df],axis = 1, ignore_index = True)

    count_df['total'] = count_df.sum(axis = 1)
    count_df['std'] = np.sqrt(count_df['total'].values);
    return count_df;

def get_files_to_process(time_when_switch=30, cluster='raven'):
    """
    cluster 1  - raven
    cluster 2 - cobra
    """
    if cluster == 'raven':
        files = []

        for k in range(1,11):
            q = time_when_switch;

            filename = 'D:\Serhii\Projects\EvoChan\Simulations\sim_results_garching\Important_Selection_escape_sims\\20210524_raven\h5_data_folder_raven\\19_05_raven_switch_at_'+str(q)+'_run_'+str(k)+'\\all_data.h5';
            files.append(filename);
        return files;
    elif cluster == 'cobra':
        files = [];
        for k in range(1,11):
            q = time_when_switch;

            filename = 'D:\Serhii\Projects\EvoChan\Simulations\sim_results_garching\Important_Selection_escape_sims\\20210524_cobra\h5_data_cobra_19_05\\19_05_cobra_switch_at_'+str(q)+'_run_'+str(k)+'\\all_data.h5';
            files.append(filename);
        return files;

def plot_probability(data,color='k',label='label not specified'):
    probability = data['total'].values/data['total'].values[0];
    err = probability*np.sqrt( (data['std'].values/data['total'].values)**2 +(data['std'].values[0]/data['total'].values[0])**2)
    x = data.index;
    print(probability)
    plt.plot(x, probability, '--', color = color, linewidth = 1.0, label = label)
    plt.fill_between(x, probability-err,probability+err, color = color, alpha = 0.5);
    plt.legend();
    return 0;

# plot
switches = [30, 35, 40, 50];
colors = ['r', 'c', 'orange', 'g'];
plt.figure(figsize=(10,10))
plt.rcParams.update({'font.size': 26})
for i in range(1):
    mutateAt = switches[i];
    files = get_files_to_process(mutateAt, 'raven');
    count_df = get_counts_structure(files);
    plot_probability(count_df,'r','tailored rescues at '+str(mutateAt))

switches = [5, 10, 15, 20];
colors = ['m', 'k', 'grey', 'y'];
files = [];
for k in range(1,11):

    filename = 'D:\Serhii\Projects\EvoChan\Simulations\sim_results_garching\Important_Selection_escape_sims\\20210810_no_switch_control\\04_08_no_switch_control_h5_data\\run_'+str(k)+'\\all_data.h5';
    files.append(filename);

count_df = get_counts_structure(files);
plot_probability(count_df,'w','no rescue')
i+=2;
for k in range(1,21):

    filename = r'D:\Serhii\Projects\EvoChan\Simulations\sim_results_garching\Important_Selection_escape_sims\20210426_large_statistics\23_04_run_'+str(k)+'\output\\all_data.h5'
    files.append(filename);

count_df = get_counts_structure(files);
plot_probability(count_df,'orange','random rescues')
plt.xlim([0,89])
plt.ylabel('Probability',fontsize=26)
plt.xlabel('Time, a.u.', fontsize = 26)
plt.savefig('probability_sim.png')
plt.show();


from rest_framework import serializers
from .models import User, Event

class UserSerializer(serializers.HyperlinkedModelSerializer):
    events = serializers.HyperlinkedRelatedField(
        view_name='event_detail',
        many=True,
        read_only=True
    )

    event_url = serializers.ModelSerializer.serializer_url_field(
        view_name='user_detail'
    )
    # event_url = serializers.HyperlinkedIdentityField(
    #     view_name='user_detail',
    #     lookup_field='username'
    # )
    class Meta:
        model = User
        fields = ('id', 'events', 'event_url', 'username')

class EventSerializer(serializers.HyperlinkedModelSerializer):
    user = serializers.HyperlinkedRelatedField(
        view_name='user_detail',
        read_only=True
    )

    user_id = serializers.PrimaryKeyRelatedField(
        queryset=User.objects.all(),
        source='user'
    )

    event_url = serializers.ModelSerializer.serializer_url_field(
        view_name='event_detail'
    )

    class Meta:
        model = Event
        fields = ('id', 'user', 'user_id', 'event_url',
                  'title', 'description', 'date',)

# Do we need a calendar serializer?
# class CalendarSerializer(serializers.HyperlinkedModelSerializer):


# author: Ziming Guo
# time: 2020/3/31
'''
    作业2_2：
    测试 进程 线程 但进程的执行效率（见test）
    检测：对于IO密集型的过程，三者的效率各是多少 ¬
'''

import time
from multiprocessing import Process
from threading import Thread

# 写一个无阻塞的IO行为
def io():
    write()
    read()

def write():
    f = open('test', 'w')
    for i in range(1800000):
        f.write("Hello World\n")
    f.close()

def read():
    f = open('test')
    lines = f.readlines()
    f.close()

# 用单进程执行 10 遍，记录执行时间（时间差）
time01 = time.time()
for i in range(9):
    io()
time02 = time.time()
sub01 = time02 - time01
print(sub01)

# 用多进程执行：十个进程，每个进程执行一遍
ls01 = []
time03 = time.time()
for i in range(9):
    p01 = Process(target=io)
    ls01.append(p01)
    p01.start()
for item in ls01:
    item.join()
time04 = time.time()
sub02 = time04 - time03
print(sub02)

# 用多线程执行：十个线程，每个线程执行一遍
ls02 = []
time05 = time.time()
for i in range(9):
    t01 = Thread(target=io)
    ls02.append(t01)
    t01.start()
for item in ls02:
    item.join()
time06 = time.time()
sub03 = time06 - time05
print(sub03)


import json
import boto3
'''
converts Dynamodb dictionary format to python dictionary format
'''
def dictToJson(d):
    e={}
    for i in d.keys():
        for j in d[i].keys():
            e[i]=d[i][j]
    e['success']=True
    return e

'''
handles input request
'''
def lambda_handler(event, context):
    client = boto3.client('dynamodb')
    d={"statusCode": 200, "body": "results"}
    event=json.loads(event['body'])
    try:
        a=client.get_item(TableName=event['flag'],Key={
            'username': {'S': event['username'].lower()}
        })
        b=dictToJson(a['Item'])
        if b['username']==event['username'].lower() and b['password']==event['password']:
            d['body']=json.dumps({'success':True})
        else:
            d['body']=json.dumps({'success':False})
    except Exception as e:
            print(e)
            d['body'] =json.dumps({'success':False})
    return d


from flask import Flask
import socket
import json
import time
import requests
import traceback
from models import Device, engine
from sqlalchemy.orm import (sessionmaker, relationship)
app = Flask(__name__)
"""
mount -o remount,rw /system # allow to write
"""

def get_host_ip():
    try:
        s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
        s.connect(('8.8.8.8', 80))
        ip = s.getsockname()[0]
    finally:
        s.close()

    return ip

@app.route('/')
def hello_world():
    return 'Hello, World!'

@app.route('/api/xtools/x008/status', methods=['POST', 'GET'])
def status():
    data = {
    	"userName": "1000000",
    	"overTime": "false",
    	"id": "1000000",
    	"time": int(time.time()) * 1000
    }
    return json.dumps(data)

@app.route('/api/xtools/x008/sysconf', methods=['POST', 'GET'])
def sysconf():
    data = {
    	"sellUrl": "http://api2.apk00888888.com/lishu008AppManager/AuToAddDays.action",
    	"status": "1",
    	"expiryTime": int(time.time()) * 1000,
    	"vip": "true",
    	"promoteUrl": "/DownLoadFileAction.action?id=1000000",
    	"msg": "未过期",
    	"id": "1000000"
    }
    return json.dumps(data)

@app.route('/api/xtools/x008/genid', methods=['POST', 'GET'])
def genid():
    data = {

    }
    # print(request)
    return json.dumps(data)

def get_device(id, imei, **kwargs):
    url = 'http://api.008shenqi.com/api/xtools/x008/device/v1'
    # id = '1022947'
    # imei = '328532726131986'
    data = {
        'id':	id,
        'imei':	imei,
        # 'id':	'1001600',
        # 'imei':	'865166026812118',
    }
    params = {
        'time': int(time.time()),
        'id':	id,
        'imei':	imei,
    }
    headers = {
        'User-Agent': 'okhttp/3.7.0',
        'Host':	'api.008shenqi.com',
        'Content-Type':	'application/x-www-form-urlencoded',
        'Accept-Encoding':	'gzip',
        'Content-Length':	'31',
    }
    r = requests.post(url, params=params, data=data, headers=headers)
    return r.json()

def keep_fetch_device_save():
    id = '1022947'
    imei = '328532726131986'
    count = 0
    # 创建DBSession类型:
    DBSession = sessionmaker(bind=engine)
    # 创建session对象:
    session = DBSession()
    while True:
        try:
            ret = get_device(id, imei)
            model = ret['model']
            getDeviceId = ret['getDeviceId']
            print('model: {}'.format(model))
            # Duplicate removal
            query = session.query(Device).filter(Device.model==model, Device.getDeviceId==getDeviceId).all()
            if query:
                print('已经存在: {}'.format(model))
            else:
                # save
                device_new = Device()
                for key, value in ret.items():
                    setattr(device_new, key, value)
                session.add(device_new)
                session.commit()
                count += 1
                print('保存成功: {}, 一共: {}'.format(device_new, count))
            time.sleep(0.2)
        except Exception as e:
            print(traceback.format_exc())
            print(ret)
            break

def main():
    try:
        print('--------------------------------')
        print('|         Ip地址: {}   |'.format(get_host_ip()))
        print('''

            第一次运行的模拟器上需要的操作步骤:
            1.打开修改hosts的app。
            2.增加或者修改一对：
                IP Address: 上面的Ip地址
                Hostname: api.008shenqi.com
        ''')
        print('--------------------------------')
        app.run(host='0.0.0.0', port=80)
    except Exception as e:
        print(e)
        a = input('回车结束')

if __name__ == "__main__":
    main()

    # id = '1022947'
    # imei = '328532726131986'
    # get_device(id, imei)

    # keep_fetch_device_save()