import json
import re
import logging
from __init__ import app

def extract_dispatch_content(text):
    pattern = r'R9\.redux\.dispatch\((\{.+?\})\)'
    matches = re.findall(pattern, text, flags=re.DOTALL)
    return matches

def add_quotes(json_str):
    # Agregar comillas a las claves de primer nivel
    regex_keys = r'(?<={|,)\s*([a-zA-Z0-9]+)\s*:'
    result = re.sub(regex_keys, r' "\1":', json_str)

    # Reemplazamos las comillas simples por dobles
    pattern = r"(?<!\w)'(?!\w)|(?<=\w)'(?!\w)|(?<!\w)'(?=\w)"
    result = re.sub(pattern, '"', result)
    return result

def get_json_from_kayak_response(response_text):
    if response_text:
        response_json = json.loads(response_text)

        # Accede a los campos del JSON y extrae el contenido de los scripts
        scripts = response_json.get('bufferedScripts')

        # Combina todas las cadenas de los scripts en una sola cadena
        combined_scripts = ' '.join(scripts)
        # Encuentra todas las ocurrencias de 'R9.redux.dispatch(...)'
        matches = extract_dispatch_content(combined_scripts)
        # Busca el contenido en el que el valor de 'type' es 'FlightResultsList'
        flight_results_list = None
        for match in matches:
            formatted_match = add_quotes(match)
            json_data = json.loads(formatted_match)
            logging.info(json_data.get('type'))
            if 'FlightResultsList' in json_data.get('type'):
                flight_results_list = json_data.get('state').get('results')
                return flight_results_list

    app.logger.debug("No se encontró un objeto con 'type' igual a 'FlightResultsList'")


import os
import re

from traitlets import List, default

from .base import BaseConverter
from ..preprocessors import (
    InstantiateTests,
    ClearOutput,
    CheckCellMetadata
)
from traitlets.config.loader import Config
from typing import Any
from ..coursedir import CourseDirectory

class GenerateSourceWithTests(BaseConverter):

    @default("permissions")
    def _permissions_default(self) -> int:
        return 664 if self.coursedir.groupshared else 644

    @property
    def _input_directory(self) -> str:
        return self.coursedir.source_directory

    @property
    def _output_directory(self) -> str:
        return self.coursedir.source_with_tests_directory

    preprocessors = List([
        InstantiateTests,
        ClearOutput,
        CheckCellMetadata
    ]).tag(config=True)

    def _load_config(self, cfg: Config, **kwargs: Any) -> None:
        super(GenerateSourceWithTests, self)._load_config(cfg, **kwargs)

    def __init__(self, coursedir: CourseDirectory = None, **kwargs: Any) -> None:
        super(GenerateSourceWithTests, self).__init__(coursedir=coursedir, **kwargs)

    def start(self) -> None:
        old_student_id = self.coursedir.student_id
        self.coursedir.student_id = '.'
        try:
            super(GenerateSourceWithTests, self).start()
        finally:
            self.coursedir.student_id = old_student_id


# CS5487 demo script for Programming Assignment 2
#
# The script has been tested with python 2.7.6
#
# It requires the following modules:
#   numpy 1.8.1
#   matplotlib v1.3.1
#   scipy 0.14.0
#   Image (python image library)

import pa2
import numpy as np
import pylab as pl
import scipy.io as sio
from PIL import Image
from Assignment2.Problem1.K_means import *
from Assignment2.Problem1.EM import *
from Assignment2.Problem1.Mean_shift import *
from Assignment2.Problem1.P1_function import *

#import Image
def demo():
    import scipy.cluster.vq as vq

    ## load and show image
    img = Image.open('../images/12003.jpg')
    pl.subplot(1, 3, 1)
    pl.imshow(img)

    ## extract features from image (step size = 7)
    X, L = pa2.getfeatures(img, 7)

    ## Call kmeans function in scipy.  You need to write this yourself!
   # C,Y = vq.kmeans2(vq.whiten(X.T), 2, iter=1000, minit='random')

    d = 4
    K = 4
    N = X.shape[1]
    Mju = RandomMju(150, 155, K, d)
    ##########Kmeans##########
    #Y = Clustering_Kmeans(Mju, X, K, N, d, 1000, 0.000000000001)

     ##########EM#########
    #d = 4
    #K = 2
    #SIGMA = RandomSIGMA(100, 110, K, d)

    #Pi = [0.1, 0.1, 0.05, 0.1, 0.1, 0.1, 0.1, 0.1, 0.05, 0.2]
    #Y = Clustering_EM(Mju, SIGMA, Pi, X, K, d, X.shape[1], 800, 0.00000000000001)
     ###################

    #####Meanshift####
    h = 45

    maxRound = 30
    limitation = 0.00001
    cluster_gap = 8
    X_peak = Peak_MeanShift(X, d, N, h, maxRound, limitation)
    cluster = Clustering_MeanShift(X, X_peak, N, d, cluster_gap)
    Y = cluster[1]
    ##########################

    # Use matlab 1-index labeling

    # make segmentation image from labels
    segm = pa2.labels2seg(Y, L)
    pl.subplot(1, 3, 2)
    pl.imshow(segm)

    # color the segmentation image
    csegm = pa2.colorsegms(segm, img)
    picname ="seastar"
    title = "Menshift, h="+str(h)
    pl.title(title)
    pl.subplot(1, 3, 3)
    pl.imshow(csegm)
    pl.savefig('/Users/gaobrook/Desktop/Clustering/problem2/'+title+'_'+picname+'.png', dpi=200)
    pl.show()

def main():
    demo()
if __name__ == '__main__':
    main()


from typing import List
from PyQt5 import QtGui
import networkx as nx
from commandbar.api import cmdutils
from commandbar.commands.cmdexc import PrerequisitesError
from commandbar.utils import objreg
from mainwindow.graph_view import GraphView
from mainwindow.mainwindow import MainWindow
from tree_covers.pygraph.metric_spaces import tree_cover_bad_pairs, tree_cover_embedding_distortion

@cmdutils.register(name='calc-stretch', instance='main-window', scope='window', maxsplit=0)
def calculate_stretch(self: MainWindow):
    """
    calculate the stretch of the first graph in the window with respect to the cover
    that is induced by all other graphs in the window
    """
    if len(self.canvases) <= 1:
        raise PrerequisitesError("Number of graph view must be more then 1 to calculate stretch")
    V = set(self.canvases[0].G.nodes)
    if not all(V.issubset(gv.G.nodes) for gv in self.canvases[1:]):
        raise PrerequisitesError("the set of verticies of the first graph must"
                                 "be a subset of the vertices of tje other graphs")
    sc = StretchCalculator(self.canvases)
    objreg.register('stretch-calculator', sc, scope='window', window=self.win_id)

class StretchCalculator:
    def __init__(self, graph_views: List[GraphView]) -> None:
        self.graph_views = graph_views
        self.bad_pairs = self.calc_and_display_strecth_cb()
        # for gv in self.graph_views:
        #     gv.G.register_on_edge_add_callback(self.calc_and_display_strecth_cb)
        #     gv.G.register_on_edge_remove_callback(self.calc_and_display_strecth_cb)
        self.last_bad_pair = None

    def calc_and_display_strecth_cb(self, *args):
        dist = tree_cover_embedding_distortion(self.graph_views[0].G, [gv.G for gv in self.graph_views[1:]])
        print(dist)
        self.bad_pairs = tree_cover_bad_pairs(self.graph_views[0].G, [gv.G for gv in self.graph_views[1:]], 1.1)
        self.bad_pairs = list(self.bad_pairs)
        print(self.bad_pairs)
        self.bad_pairs = iter(self.bad_pairs)
        return self.bad_pairs

    @cmdutils.register(instance='stretch-calculator', name='next-stetch-pair', scope='window')
    def next_pair(self):
        """
        displays the next pair that has high stretch
        and its respective paths in each graph view
        """
        if self.last_bad_pair:
            u, v = self.last_bad_pair
            for gv in self.graph_views:
                p = nx.shortest_path(gv.G, u, v)
                gv.draw_edges(zip(p, p[1:]))
                gv.draw_vertices(p)
        try:
            u, v = next(self.bad_pairs)
            self.last_bad_pair = u, v
            for gv in self.graph_views:
                p = nx.shortest_path(gv.G, u, v)
                red_pen = QtGui.QPen(QtGui.QColor('red'))
                gv.draw_edges(zip(p, p[1:]), red_pen, None)
                gv.draw_vertices(p, red_pen)
        except StopIteration:
            raise PrerequisitesError("Out of bad pairs")


# -*- coding: utf-8 -*-
# This script goes through USFM files, generating a list of verses that contain properly marked footnotes.
# Reports errors to stderr and issues.txt.
# Set source_dir and usfmVersion to run.

# Global variables
source_dir = r'C:\DCS\Portuguese\pt-br_ulb'
usfmVersion = 2     # if version 3.0 or greater, tolerates unknown tokens and verse fragments

import os
import sys
import parseUsfm
import io
import re
import json
if usfmVersion >= 3.0:
    import usfm_utils

vv_re = re.compile(r'([0-9]+)-([0-9]+)')

class State:
    IDs = []
    ID = ""
    chapter = 0
    verse = 0
    reference = ""
    footnoteRefs = list()

    # Resets state data for a new book
    def addID(self, id):
        State.IDs.append(id)
        State.ID = id
        State.chapter = 0
        State.verse = 0
        State.reference = id

    def getIDs(self):
        return State.IDs

    def addChapter(self, c):
        State.chapter = int(c)
        State.verse = 0
        State.reference = State.ID + " " + c

    def addVerse(self, v):
        State.verse = int(v)
        State.reference = State.ID + " " + str(State.chapter) + ":" + v

    # Adds the current reference to the list of footnote references
    def addFootnote(self):
        if self.reference not in State.footnoteRefs:
            self.footnoteRefs.append(self.reference)

    # Returns list of footnote references as a json string
    def getFootnoteReferences(self):
        return json.dumps(self.footnoteRefs)

    def countFootnotes(self):
        return len(self.footnoteRefs)

def shortname(longpath):
    shortname = longpath
    if source_dir in longpath:
        shortname = longpath[len(source_dir)+1:]
    return shortname

def takeID(id):
    state = State()
    if len(id) < 3:
        reportError("Invalid ID: " + id)
    id = id[0:3].upper()
    if id in state.getIDs():
        reportError("Duplicate ID: " + id)
    state.addID(id)

def takeC(c):
    state = State()
    state.addChapter(c)

# Receives a string containing a verse number or range of verse numbers.
# Reports errors related to the verse number(s), such as missing or duplicated verses.
def takeV(vstr):
    state = State()
    vlist = []
    if vstr.find('-') > 0:
        vv_range = vv_re.search(vstr)
        if vv_range:
            vn = int(vv_range.group(1))
            vnEnd = int(vv_range.group(2))
            while vn <= vnEnd:
                vlist.append(vn)
                vn += 1
        else:
            reportError("Problem in verse range near " + State.reference)
    else:
        vlist.append(int(vstr))

    for vn in vlist:
        v = str(vn)
        state.addVerse(str(vn))

# Writes error message to stderr and to issues.txt.
def reportError(msg):
    try:
        sys.stderr.write(msg + "\n")
    except UnicodeEncodeError as e:
        state = State()
        sys.stderr.write(state.reference + ": (Unicode...)\n")

# Returns true if token is a countable part of a footnote
def isFootnote(token):
    # return token.isF_S() or token.isF_E() or token.isFR() or token.isFR_E() or token.isFT() or token.isFP() or token.isFE_S() or token.isFE_E()
    return token.isF_S() or token.isF_E()

def take(token):
    global usfmVersion

    state = State()
    if isFootnote(token):
        state.addFootnote()
    if token.isID():
        takeID(token.value)
    elif token.isC():
        if not state.ID:
            reportError("Missing book ID: " + state.reference)
            sys.exit(-1)
        takeC(token.value)
    elif token.isV():
        takeV(token.value)

# Corresponding entry point in tx-manager code is verify_contents_quiet()
def processFile(path):
    input = io.open(path, "tr", 1, encoding="utf-8-sig")
    str = input.read(-1)
    input.close()
    if usfmVersion >= 3.0:
        str = usfm_utils.usfm3_to_usfm2(str)

    print("CHECKING " + shortname(path))
    sys.stdout.flush()
    for token in parseUsfm.parseString(str):
        take(token)
    state = State()
    state.addID("")
    sys.stderr.flush()

# Verifies all .usfm files under the specified folder.
def processDir(dirpath):
    for f in os.listdir(dirpath):
        if f[0] != '.':         # ignore hidden files
            path = os.path.join(dirpath, f)
            if os.path.isdir(path):
                # It's a directory, recurse into it
                processDir(path)
            elif os.path.isfile(path) and path[-3:].lower() == 'sfm':
                processFile(path)

# Writes list of footnote references to a file.
def dumpFootnoteReferences():
    state = State()
    refs = state.getFootnoteReferences()
    if refs:
        path = os.path.join(source_dir, "footnotedVerses.json")
        footnoteFile = io.open(path, "tw", encoding='utf-8', newline='\n')
        footnoteFile.write(refs)
        footnoteFile.close()
    sys.stdout.write("Found " + str(state.countFootnotes()) + " footnotes.\n")

if __name__ == "__main__":
    if len(sys.argv) > 1 and sys.argv[1] != 'hard-coded-path':
        source_dir = sys.argv[1]

    if os.path.isdir(source_dir):
        processDir(source_dir)
    elif os.path.isfile(source_dir):
        path = source_dir
        source_dir = os.path.dirname(path)
        processFile(path)
    else:
        reportError("File not found: " + source_dir)

    dumpFootnoteReferences()


# -*- coding: utf-8 -*-
# +
import argparse
import datetime
import random
import time
import pandas as pd
from pathlib import Path

import torch
import torchvision.transforms as standard_transforms
import numpy as np

from PIL import Image
import cv2
from crowd_datasets import build_dataset
from engine import *
from models import build_model
import os
import warnings
warnings.filterwarnings('ignore')

def get_args_parser():
    parser = argparse.ArgumentParser('Set parameters for P2PNet evaluation', add_help=False)

    # * Backbone
    parser.add_argument('--backbone', default='vgg16_bn', type=str,
                        help="name of the convolutional backbone to use")

    parser.add_argument('--row', default=2, type=int,
                        help="row number of anchor points")
    parser.add_argument('--line', default=2, type=int,
                        help="line number of anchor points")

    parser.add_argument('--output_dir', default='./',
                        help='path where to save')
    parser.add_argument('--weight_path', default='./weights/SHTechA.pth',
                        help='path where the trained weights saved')

    parser.add_argument('--gpu_id', default=0, type=int, help='the gpu used for evaluation')

    return parser

def main(args, debug=False):
    start=time.time()

    os.environ["CUDA_VISIBLE_DEVICES"] = '{}'.format(args.gpu_id)

    print(args)
    device = torch.device('cuda')
    # get the P2PNet
    model = build_model(args)
    # move to GPU
    model.to(device)
    # load trained model
    if args.weight_path is not None:
        checkpoint = torch.load(args.weight_path, map_location='cpu')
        model.load_state_dict(checkpoint['model'])
    # convert to eval mode
    model.eval()
    # create the pre-processing transform
    transform = standard_transforms.Compose([
        standard_transforms.ToTensor(),
        standard_transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),
    ])
    url = "https://889962c23ee6.ap-northeast-2.playback.live-video.net/api/video/v1/ap-northeast-2.196202674046.channel.ppLGhcD0q6ZR.m3u8"
    cap=cv2.VideoCapture(url)
    fourcc = cv2.VideoWriter_fourcc(*'FMP4')

    fps = cap.get(cv2.CAP_PROP_FPS)
    w = cap.get(cv2.CAP_PROP_FRAME_WIDTH) #원본 동영상 크기 정보 받아옴
    h = cap.get(cv2.CAP_PROP_FRAME_HEIGHT)

    out = cv2.VideoWriter('output.mp4', fourcc, 30, (int(w//128*128),int(h//128*128)))

    while True:
      hasFrame, img_frame=cap.read() #프레임 읽어옴
      if not hasFrame: #더이상 받아올 프레임 없는 경우
        break

      img_raw = Image.fromarray(img_frame).convert('RGB')
      # round the size
      width, height = img_raw.size
      new_width = width // 128 * 128
      new_height = height // 128 * 128
      img_raw = img_raw.resize((new_width, new_height), Image.ANTIALIAS)
      # pre-proccessing
      img = transform(img_raw)

      samples = torch.Tensor(img).unsqueeze(0)
      samples = samples.to(device)
      # run inference
      outputs = model(samples)
      outputs_scores = torch.nn.functional.softmax(outputs['pred_logits'], -1)[:, :, 1][0]

      outputs_points = outputs['pred_points'][0]

      threshold = 0.5
      # filter the predictions
      points = outputs_points[outputs_scores > threshold].detach().cpu().numpy().tolist()
      predict_cnt = int((outputs_scores > threshold).sum())

      outputs_scores = torch.nn.functional.softmax(outputs['pred_logits'], -1)[:, :, 1][0]

      outputs_points = outputs['pred_points'][0]
      # draw the predictions
      size = 5
      img_to_draw = cv2.cvtColor(np.array(img_raw), cv2.COLOR_RGB2BGR)
      for p in points:
          img_to_draw = cv2.circle(img_to_draw, (int(p[0]), int(p[1])), size, (0, 0, 255), -1)
      img_to_draw=cv2.cvtColor(np.array(img_to_draw), cv2.COLOR_BGR2RGB)
      out.write(img_to_draw)

      if cv2.waitKey(1) & 0xFF == ord('q'):
        break

      if predict_cnt>=5:
        print(predict_cnt)

        outputs_df = pd.DataFrame(points, columns=['width', 'height'])
        outputs_df.to_csv('tmp_points.csv', encoding='utf-8')

    print("original size: ", width, height)
    print("resized size: ", new_width, new_height)

    cap.release()
    out.release()
    print(time.time()-start)

if __name__ == '__main__':
    parser = argparse.ArgumentParser('P2PNet evaluation script', parents=[get_args_parser()])
    args = parser.parse_args()
    main(args)
# -


import cv2
import numpy as np

img = cv2.imread('/home/kanish/Documents/ICR advanced forms/Advanced handwritting samples/athul_scanned/525/525_2.png', 0)

kernel = np.ones((5, 1), np.uint8)
erosion = cv2.erode(img, kernel, iterations=1)

cv2.imwrite('morphex.png', erosion)

cv2.imshow('gray', erosion)
cv2.waitKey(0)


import requests
from bs4 import BeautifulSoup as soup

def requesting_ip():
    HEADERS = {
        'Accept':'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
        #'Accept-Encoding':'gzip, deflate, br',
        'Accept-Language':'en-US,en;q=0.5',
        'Connection':'keep-alive',
        'Upgrade-Insecure-Requests':'1',
        'User-Agent':'Mozilla/5.0 (Windows NT 6.1; rv:60.0) Gecko/20100101 Firefox/60.0',
    }

    url = 'https://www.duckduckgo.com/html?q=my+ip'

    if url:
        r = requests.get(url, headers=HEADERS)
        page = soup(r.text, 'html5lib')
        page = str(page).splitlines()
        return page
    elif url_check == False:
        print('< check url >')

def parsing_ip(page):
    for line in page:
        if 'Your IP address is ' in line and ' in <a href="http' in line:
            line = line.replace('Your','\nYour').splitlines()[1]
            line = line.split(' ')
            print(f'{line[0]} {line[1]} {line[2]} {line[3]} {line[4]} ')

def main():
    page = requesting_ip()
    if page:
        parsing_ip(page)

if __name__ == '__main__':
    main()


#!/usr/bin/python3

from typing import List
import json
from bplib.butil import TreeNode, arr2TreeNode, btreeconnect

class Solution:
    def intersect(self, nums1: List[int], nums2: List[int]) -> List[int]:
        st = {}
        for n in nums1:
            if n not in st:
                st[n] = 0
            st[n] += 1
        ans = []
        for n in nums2:
            if n in st and st[n] > 0:
                st[n] -= 1
                ans.append(n)
        return ans

arr1 = json.loads(input())
arr2 = json.loads(input())

sol = Solution()
print(sol.intersect(arr1, arr2))


# importing Numpy package
import numpy as np

# To declare symbol variables x and y
from sympy import symbols

 #  You can't use numpy arrays for symbolical calculations with sympy. Instead, use a sympy Matrix
#    (https://stackoverflow.com/questions/68589864/matrix-determinant-symbolic-in-python)
import sympy as sp

# creating a 3X3 Numpy matrix
n_array = np.array([[55, 25, 15],
                    [30, 44, 2],
                    [11, 45, 77]])

# Displaying the Matrix
print("Numpy Matrix is:")
print(n_array)

# calculating the determinant of matrix
det = np.linalg.det(n_array)

print("\nDeterminant of given 3X3 square matrix:")
print(int(det))

# FOR DETERMINANTS WITH SYMBOLS VARIABLES

# Problemas de clase 1 ejercicio 2

# Declaring symbol variables x and y
x = sp.symbols('x')
y = sp.symbols('y')
z = sp.symbols('z')

# creating a 3X3 sp matrix
matrix = sp.Matrix(([1, x, x**2],
                    [1, y, y**2],
                    [1, z, z**2]))

# Displaying the Matrix
print("Matrix is:")
print(matrix)

# calculating the determinant of sp nmatrix
det3 = sp.det(matrix)

print("\nDeterminant of given 3X3 square matrix:")
print((det3))


import os
import traceback
from prefect import flow
from conf.config import DATA_DIR, DATASET_EXCEL_LINKS
from src.utils import write_local_to_parquet
from src.flows.pipeline_components import ingest_data_from_list_files, load_csv_dataset, save_dataset_to_csv, upload_to_gcs_bucket, write_data_google_bq

@flow(name="end_to_end_pipeline_from_http_to_bq",
      description="Flow that download the dataset in excel format given a list of links (excel), "
                  "Combine the data, save it to csv, convert to parquet and then upload to google cloud bucket",
      log_prints=True)
def end_to_end_pipeline_from_http_to_bq(filename_arr_path: list = DATASET_EXCEL_LINKS,
                                        name_output_dataset="global_terrorism_db"):
    """
    End to end pipeline
    Download files given an array of links > convert to csv > save to parquet format > upload to google cloud bucket
    > upload to google big query
    :param filename_arr_path:
    :param name_output_dataset:
    :return:
    """
    try:
        # 0. Check beforehand if there is CSV dataset so that we don't download again
        filename_csv = name_output_dataset + '.csv'
        path_filename_csv = DATA_DIR.joinpath(filename_csv)
        if not os.path.isfile(path_filename_csv):
            # 1. Download excel data from links (github)
            combined_data_df = ingest_data_from_list_files(filename_arr_path)
            # 2. Convert data to csv
            path_filename_csv = save_dataset_to_csv(combined_data_df, name_output_dataset)
        else:
            print("CSV file already exists")
        # 3. Load the csv dataset freshly created
        data_df = load_csv_dataset(path_filename_csv)
        # 4. save to parquet format
        dataset_parquet_path = os.path.splitext(path_filename_csv)[0] + ".parquet"
        write_local_to_parquet(data_df, dataset_parquet_path)
        # 5. Upload to GCS
        upload_to_gcs_bucket(dataset_parquet_path)
        # 6. Upload to Google Big query
        write_data_google_bq(data_df)

    except Exception as ex:
        print("Exception raised: ", ex)
        print(traceback.format_exc())
        exit(-1)

if __name__ == '__main__':
    name_output_data = "global_terrorism_db"
    filename_arr_path = DATASET_EXCEL_LINKS
    end_to_end_pipeline_from_http_to_bq(filename_arr_path=DATASET_EXCEL_LINKS, name_output_dataset=name_output_data)


from pyspark.sql import SparkSession
from pyspark.sql import S
Logger.getLogger("org").setLevel(Level.ERROR)

sparkConf = newSparkConf()
sparkConf.set("spark.app.name", "My Application 1")
sparkConf.set("spark.master", "local[2]")

spark = SparkSession.builder()\
    .config(sparkConf)\
    .getOrCreate()

DDLString = "country String, weeknum Int, numinvoices Int, totalquantity Int, invoicevalue Double"

input = spark.read\
    .format("csv")\
    .schema(DDLString)\
    .option("path", "C:/Users/chnis/Downloads/Big Data/week11/windowdata.csv")\
    .load

input.write\
    .format("avro")\
    .partitionBy("country")\
    .mode("Overwrite")\
    .option("path", "C:/Users/chnis/Downloads/Big Data/week11/windowdata_output_avro")\
    .save()


from django.conf.urls.defaults import *
from django.conf import settings
from django.contrib import admin
from seishinkan.website.feeds import NewsFeed, TerminFeed

admin.autodiscover()

feeds = {
  'termine': TerminFeed,
  'news': NewsFeed,
}

urlpatterns = patterns('',
    (r'^i18n/', include('django.conf.urls.i18n')),
    (r'^verwaltung/doc/', include('django.contrib.admindocs.urls')),
    (r'^verwaltung/', include(admin.site.urls)),
    (r'^feed/termine/$', TerminFeed()),
    (r'^feed/news/$', NewsFeed()),
)

if 'rosetta' in settings.INSTALLED_APPS:
    urlpatterns += patterns('',
        url(r'^rosetta/', include('rosetta.urls')),
  )

# Simple Generic Views
urlpatterns += patterns('django.views.generic.simple',
    (r'^accounts/$', 'redirect_to', {'url': '/'}),
)

# Seishinkan Website Views
urlpatterns += patterns('seishinkan.website.views',
    (r'^$', 'index'),
    (r'^login/$', 'seishinkan_login'),
    (r'^lang/(.*)/$', 'set_lang'),
    (r'^logout/$', 'seishinkan_logout'),
    (r'^seite/(\d+)/$', 'index'),
    (r'^links/$', 'links'),
    (r'^kontakt/$', 'kontakt'),
    (r'^news/$', 'news'),
    (r'^news/(\d+)/$', 'news'),
    (r'^news/archiv/$', 'news'),
    (r'^termin/$', 'termin'),
    (r'^termin/(\d+)/$', 'termin'),
    (r'^termin/archiv/$', 'termin'),
    (r'^termine/$', 'termin'),
    (r'^termine/archiv/$', 'termin'),
    (r'^impressum/$', 'impressum'),
    (r'^info/$', 'info'),
    (r'^sysinfo/$', 'sysinfo'),
    (r'^videos/$', 'video'),
    (r'^bilder/$', 'picasa_albums'),
    (r'^bilder/(.+)/$', 'picasa_photos'),
    (r'^downloads/$', 'downloads'),
    # (r'^videos/(.+)/$', 'video'),
    (r'^email/$', 'mailinglist'),
    (r'^log/$', 'admin_log'),
    (r'^permissions/$', 'permissions'),
    (r'^graduierungen/$', 'graduierungen'),
    (r'^mitglieder/$', 'mitgliederlisten'),
    (r'^mitglieder/csv/$', 'mitglieder_csv'),
    (r'^mitglieder/csv/(\d+)/$', 'mitglieder_csv'),
    (r'^mitglieder/xls/$', 'mitglieder_xls'),
    (r'^mitglieder/xls/(\d+)/$', 'mitglieder_xls'),
    (r'^trainerliste/(\d+)/(\d+)/$', 'trainerliste_xls'),
    (r'^teilnehmerliste/(\d+)/(\d+)/$', 'teilnehmerliste_xls'),

    # dynamic_url muss am Ende stehen
    (r'^(.+)/$', 'dynamic_url'),
)

if settings.DEBUG:
    urlpatterns += patterns('',
        (r'^static/(?P<path>.*)$', 'django.views.static.serve', {'document_root': settings.MEDIA_ROOT}),
    )

handler404 = 'seishinkan.website.views.my_404'


# -*- coding: utf-8 -*-
import scrapy
from scrapy import Request
from lxml import etree
from WaiBaoSpider.utils.csvWriter import CSVDumper
from WaiBaoSpider.utils.base import unicode_body, deal_ntr
import os

class LuAnSpider(scrapy.Spider):
    name = "luan"
    # base_url = "http://www.luan.gov.cn/nocache/supervision/?product_id=3&page={}"
    base_url = "http://www.luan.gov.cn/zxly/?type=product&page={}"
    data_path = os.getcwd() + "/WaiBaoSpider/data/"
    if os.path.exists(data_path):
        pass
    else:
        os.mkdir(data_path)
    dump_list = CSVDumper(data_path + "%s_list.csv" % name)
    dump_detail = CSVDumper(data_path + "%s_detail.csv" % name)
    headers = {
        "User-Agent": "Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.98 Safari/537.36",
    }

    # custom_settings = {
    # }

    def start_requests(self):
        for i in range(1, 915):
            # for i in range(2, 4):
            url = self.base_url.format(i)
            print(url)
            yield Request(url, headers=self.headers, callback=self.parse_list)

    # def parse_list(self, response):
    #     body = unicode_body(response)
    #     html = etree.HTML(body)
    #     lines = html.xpath("//table[@class='is-xjnr']/tr")
    #     print(len(lines))
    #     for info in lines:
    #         item = {}
    #         item[u"类别"] = info.xpath("./td[1]/text()")[0].strip() if info.xpath("./td[1]/text()") else ""
    #         title = info.xpath("./td[3]/a//text()") if info.xpath("./td[3]/a//text()") else []
    #         item[u"来信标题"] = deal_ntr("".join(title).strip())
    #         item[u"来信时间"] = info.xpath("./td[5]/text()")[0].strip() if info.xpath("./td[5]/text()") else ""
    #         item[u"处理状态"] = info.xpath("./td[7]/span/text()")[0].strip() if info.xpath("./td[7]/span/text()") else ""
    #         item[u"浏览次数"] = info.xpath("./td[9]/text()")[0].strip() if info.xpath("./td[9]/text()") else ""
    #         item[u"回复评价"] = info.xpath("./td[11]/a/text()")[0].strip() if info.xpath("./td[11]/a/text()") else ""
    #         item[u"链接"] = "http://www.luan.gov.cn{}".format(
    #             info.xpath("./td[3]/a/@href")[0].strip() if info.xpath("./td[3]/a/@href") else "")
    #         if info.xpath("./td[3]/a/span/text()"):
    #             item[u"是否公开"] = u"不公开"
    #             item_detail = {
    #                 u"链接": item[u"链接"],
    #                 u"标题": item[u"来信标题"],
    #                 u"来信人": u"",
    #                 u"来信时间": item[u"来信时间"],
    #                 u"处理情况": item[u"处理状态"],
    #                 u"督办部门": u"",
    #                 u"问题类别": u"",
    #                 u"浏览": item[u"浏览次数"],
    #                 u"来信内容": u"",
    #                 u"回复内容": u"",
    #                 u"回复单位": u"",
    #                 u"回复时间": u"",
    #                 u"是否公开": u"否",
    #             }
    #             self.dump_detail.process_item(item_detail)
    #         else:
    #             item[u"是否公开"] = u"公开"
    #             yield Request(item[u"链接"], headers=self.headers, callback=self.parse_detail,
    #                           meta={"url": item[u"链接"], "llcs": item[u"浏览次数"], "clzt": item[u"处理状态"]})
    #         self.dump_list.process_item(item)

    def parse_list(self, response):
        body = unicode_body(response)
        html = etree.HTML(body)
        lines = html.xpath("//ul[@class='is-listnews']/li")
        print(len(lines))
        for info in lines:
            item = {}
            item[u"来信标题"] = info.xpath("./a/text()")[0].strip() if info.xpath("./a/text()") else ""
            item[u"来信时间"] = info.xpath("./span/text()")[0].strip() if info.xpath("./span/text()") else ""
            item[u"链接"] = "http://www.luan.gov.cn{}".format(
                info.xpath("./a/@href")[0].strip() if info.xpath("./a/@href") else "")
            self.dump_list.process_item(item)
            yield Request(item[u"链接"], headers=self.headers, callback=self.parse_detail,
                          meta={"url": item[u"链接"]})

    def parse_detail(self, response):
        body = unicode_body(response)
        data = response.meta
        html = etree.HTML(body)
        item = {}
        item[u"链接"] = data["url"]
        item[u"标题"] = html.xpath("//div[@class='is-newstitle']/text()")[0].strip() if html.xpath(
            "//div[@class='is-newstitle']/text()") else ""
        item[u"来信人"] = html.xpath("//div[@class='is-mailinfo']/text()[1]")[0].strip().replace(u"来信人：",
                                                                                u"") if html.xpath(
            "//div[@class='is-mailinfo']/text()[1]") else ""
        item[u"来信时间"] = html.xpath("//div[@class='is-mailinfo']/text()[2]")[0].strip().replace(u"来信时间：",
                                                                                u"") if html.xpath(
            "//div[@class='is-mailinfo']/text()[2]") else ""
        item[u"处理情况"] = html.xpath("//span[@class='orange state']/text()")[0].strip() if html.xpath(
            "//span[@class='orange state']/text()") else ""
        dbbm = html.xpath("//div[@class='is-mailinfo']/span[4]/text()[1]")[0].strip().replace(u"|",
                                                                                u"").replace(
            u"督办部门：", u"") if html.xpath("//div[@class='is-mailinfo']/span[4]/text()[1]") else ""
        item[u"督办部门"] = deal_ntr(dbbm)
        item[u"问题类别"] = html.xpath("//div[@class='is-mailinfo']/span[4]/span[@class='red'][1]/text()")[
            0].strip() if html.xpath("//div[@class='is-mailinfo']/span[4]/span[@class='red'][1]/text()") else ""
        item[u"浏览"] = html.xpath("//div[@class='is-mailinfo']/span[4]/span[@class='red'][2]/text()")[
            0].strip() if html.xpath("//div[@class='is-mailinfo']/span[4]/span[@class='red'][2]/text()") else ""
        lxnr = html.xpath("//div[@class='is-mailwen']/p//text()") if html.xpath(
            "//div[@class='is-mailwen']/p//text()") else []
        item[u"来信内容"] = deal_ntr("".join(lxnr))
        hf_content = html.xpath("//div[@class='is-hfcontent']//text()") if html.xpath(
            "//div[@class='is-hfcontent']//text()") else []
        item[u"回复内容"] = deal_ntr("".join(hf_content))
        item[u"回复单位"] = html.xpath("//div[@class='is-mialhf']/h1/span[2]/text()")[0].strip().replace(u"回复单位：",
                                                                                u"") if html.xpath(
            "//div[@class='is-mialhf']/h1/span[2]/text()") else ""
        item[u"回复时间"] = html.xpath("//div[@class='is-mialhf']/h1/span[1]/text()")[0].strip().replace(u"回复时间：",
                                                                                u"") if html.xpath(
            "//div[@class='is-mialhf']/h1/span[1]/text()") else ""
        self.dump_detail.process_item(item)


import numpy as np
from scipy import interpolate
class SkeletonASCIIPhraser(object):

    def __init__(self, export_path):

        self.__export_path = export_path

    def save(self, beams):
        file = open(self.__export_path, 'w')
        self.__plot_analytical_skeleton_beams(file, beams)
        file.close()

    def __plot_analytical_skeleton_beams(self, file, beams):
        # Save the ids and the export to the ids in a dictionary
        point_ids = {}
        vector_ids = {}
        spline_ids = {}
        curve_ids = {}
        cross_sections_ids = {}
        beam_ids = {}
        junction_connections = {}
        # ID counters
        point_id = 0
        spline_id = 0
        curve_id = 0
        vector_id = 0
        cross_section_id = 0
        beam_id = 0
        junction_id_point_list = {}
        start_end_cross_section_correspondence = {}

        for beam in beams:
            spline = beam.spline
            # calculate points
            point_id += 1
            p_jun_start_id = point_id
            point_ids[point_id] = [spline.x[0], spline.y[0], spline.z[0]]
            point_id += 1
            # Save point to point id
            point_ids[point_id] = [spline.x[-1], spline.y[-1], spline.z[-1]]
            p_jun_end_id = point_id
            # Calculate vectors
            vector_id += 1
            # Build up vector + id
            n_x1 = spline.degree * (spline.x[0] - spline.x[1])
            n_y1 = spline.degree * (spline.y[0] - spline.y[1])
            n_z1 = spline.degree *  (spline.z[0] - spline.z[1])
            norm = np.sqrt(n_x1 ** 2 + n_y1 ** 2 + n_z1 ** 2)
            norm_vec = 1.0 / norm * np.array([n_x1, n_y1, n_z1])
            vector_ids[vector_id] = [norm_vec[0], norm_vec[1], norm_vec[2]]
            vector_id += 1
            n_x1 = spline.degree * (spline.x[-1] - spline.x[-2])
            n_y1 = spline.degree * (spline.y[-1] - spline.y[-2])
            n_z1 = spline.degree * (spline.z[-1] - spline.z[-2])
            norm = np.sqrt(n_x1 ** 2 + n_y1 ** 2 + n_z1 ** 2)
            norm_vec = 1.0 / norm * np.array([n_x1, n_y1, n_z1])
            vector_ids[vector_id] = [norm_vec[0], norm_vec[1], norm_vec[2]]
            # Calculate spline and curve
            spline_id += 1
            spline_ids[spline_id] = [point_id -1, point_id, vector_id -1, vector_id]
            start_end_cross_section_correspondence[p_jun_start_id] = spline_id
            start_end_cross_section_correspondence[p_jun_end_id] = spline_id
            curve_id += 1
            curve_ids[curve_id] = [spline_id]
            # ---------- Cross section location start
            cut_times = beam.cut_times
            tck = [np.array(spline.t),
                   [np.array(spline.x), np.array(spline.y), np.array(spline.z)],
                   spline.degree]
            [x_coordinates, y_coordinates, z_coordinates] = interpolate.splev(cut_times, tck)
            [x_normals, y_normals, z_normals] = interpolate.splev(cut_times, tck, der=1)
            point_id += 1
            point_ids[point_id] = [x_coordinates[0], y_coordinates[0], z_coordinates[0]]
            # Cross section location end
            point_id += 1
            point_ids[point_id] = [x_coordinates[1], y_coordinates[1], z_coordinates[1]]
            p_c_start = point_id -1
            p_c_end = point_id

            # ----------- Cross section normal vector
            vector_id += 1
            vector_ids[vector_id] = [x_normals[0], y_normals[0], z_normals[0]]
            vector_id += 1
            vector_ids[vector_id] = [x_normals[1], y_normals[1], z_normals[1]]
            v_start = vector_id -1
            v_end = vector_id
            # Calculate beam
            beam_id += 1
            beam_id_list = []
            # Ad spline for beam
            beam_id_list.append(curve_id)
            # Add al valid cross section which are found
            # Calculate start cross sections
            if beam.crossSection1[0] != None:
                [xx, yy, zz] = beam.crossSection1[0]
                splines = []
                for id in range(len(xx)):
                    point_id += 1
                    point_ids[point_id] = [xx[id], yy[id], zz[id]]
                    if id == 0:
                        spline_start_point = point_id
                    if id != 0:
                        spline_id += 1
                        spline_ids[spline_id] = [point_id - 1, point_id]
                        splines.append(spline_id)
                    # Last point and close polygon
                    if id == len(xx) - 1:
                        spline_id += 1
                        spline_ids[spline_id] = [point_id, spline_start_point]
                        splines.append(spline_id)
                curve_id += 1
                curve_ids[curve_id] = splines
                beam_id_list.append(curve_id)
                cross_section_id += 1
                cross_sections_ids[cross_section_id] = [p_c_start, v_start, curve_id]
            # Calculate end cross sections
            if beam.crossSection2[0] != None:
                [xx, yy, zz] = beam.crossSection2[0]
                splines = []
                for id in range(len(xx)):
                    point_id += 1
                    point_ids[point_id] = [xx[id], yy[id], zz[id]]
                    if id == 0:
                        spline_start_point = point_id
                    if id != 0:
                        spline_id += 1
                        spline_ids[spline_id] = [point_id - 1, point_id]
                        splines.append(spline_id)
                    # Last point
                    if id == len(xx) - 1:
                        spline_id += 1
                        spline_ids[spline_id] = [point_id, spline_start_point]
                        splines.append(spline_id)
                curve_id += 1
                curve_ids[curve_id] = splines
                beam_id_list.append(curve_id)
                cross_section_id += 1
                cross_sections_ids[cross_section_id] = [p_c_end, v_end, curve_id]
            beam_ids[beam_id] = beam_id_list
            # Save the junction information
            end_marker = spline.end_connection_id
            start_marker = spline.start_connection_id
            if end_marker != None:
                if tuple(end_marker) in junction_connections:
                    junction_connections[tuple(end_marker)].append(cross_section_id)
                else:
                    junction_connections[tuple(end_marker)] = [cross_section_id]

                # Check if junction point is alreadcy set and then insert them
                if tuple(end_marker) in junction_id_point_list:
                    point_id_jun_end = junction_id_point_list[tuple(end_marker)] = point_id
                else:
                    point_id += 1
                    point_ids[point_id] = [end_marker[0], end_marker[1], end_marker[2]]
                    point_id_jun_end = point_id
                spline_id += 1
                spline_ids[spline_id] = [p_jun_end_id, point_id_jun_end]
                curve_id += 1
                curve_ids[curve_id] = spline_id
                # Create spline between junction and beam

            if start_marker != None:
                if tuple(start_marker) in junction_connections:
                    junction_connections[tuple(start_marker)].append(cross_section_id - 1)
                else:
                    junction_connections[tuple(start_marker)] = [cross_section_id - 1]
                    # Check if junction point is alreadcy set and then insert them

                    if tuple(start_marker) in junction_id_point_list:
                        point_id_jun_start = junction_id_point_list[tuple(start_marker)] = point_id
                    else:
                        point_id += 1
                        point_ids[point_id] = [start_marker[0], start_marker[1], start_marker[2]]
                        point_id_jun_start = point_id
                    spline_id += 1
                    spline_ids[spline_id] = [p_jun_start_id, point_id_jun_start]
                    curve_id += 1
                    curve_ids[curve_id] = spline_id
                    # Create spline between junction and beam

        file.write('POINT \n')
        for point_key in point_ids:
            if point_key in start_end_cross_section_correspondence:
                file.write(
                    f'{point_key}, {point_ids[point_key][0]}, {point_ids[point_key][1]}, {point_ids[point_key][2]},'
                    f' {start_end_cross_section_correspondence[point_key]} \n')
            else:
                file.write(f'{point_key}, {point_ids[point_key][0]}, {point_ids[point_key][1]}, {point_ids[point_key][2]} \n')
        file.write('ENDPOINT \n')

        file.write('VECTOR \n')
        for vector_key in vector_ids:
            file.write(f'{vector_key}, {vector_ids[vector_key][0]}, {vector_ids[vector_key][1]}, {vector_ids[vector_key][2]} \n')
        file.write('ENDVECTOR \n')

        file.write('SPLINE \n')
        for spline_key in spline_ids:
            if len(spline_ids[spline_key]) == 2:
                file.write(f'{spline_key}, {spline_ids[spline_key][0]}, {spline_ids[spline_key][1]} \n')
            else:
                file.write(
                    f'{spline_key}, {spline_ids[spline_key][0]}, {spline_ids[spline_key][1]}, {spline_ids[spline_key][2]}, {spline_ids[spline_key][3]} \n')
        file.write('ENDSPLINE \n')

        file.write('CURVE \n')
        for curve_key in curve_ids:
            spline_ids = curve_ids[curve_key]
            if type(spline_ids) == int:
                file.write(f'{curve_key}, {spline_ids}\n')
                continue
            if len(spline_ids) == 0: continue
            file.write(f'{curve_key}')
            for sp_id in spline_ids:
                file.write(f', {sp_id}')
            file.write("\n")
        file.write('ENDCURVE \n')

        file.write('SECTION \n')
        for cross_key in cross_sections_ids:
            file.write(f'{cross_key}, {cross_sections_ids[cross_key][0]}, {cross_sections_ids[cross_key][1]}, {cross_sections_ids[cross_key][2]}')
            file.write("\n")
        file.write('ENDSECTION \n')

        file.write('BEAM \n')
        for beam_key in beam_ids:
            file.write(f'{beam_key}')
            for beam_id_value in beam_ids[beam_key]:
                file.write(f', {beam_id_value}')
            file.write("\n")
        file.write('ENDBEAM \n')

        file.write('JUNCTION \n')
        junction_id = 0
        for junction_key, beam_ids in junction_connections.items():
            if  len(beam_ids) < 2:
                continue
            junction_id += 1
            file.write(f'{junction_id}')
            for cross_key in cross_sections_ids:
                for beam_id in beam_ids:
                    if beam_id == cross_key:
                        print(cross_key)
                        print(cross_sections_ids[cross_key][2])
                        file.write(f', {cross_sections_ids[cross_key][2]}')
            file.write("\n")
        file.write('ENDJUNCTION \n')
        file.close()


import datatable as dt
import pytest
import random
import re
from datatable import stype, f
from datatable.internal import frame_integrity_check
from tests import noop, random_string, assert_equals

def test_issue_1912():
    # Check that the expression `A==None` works if A is a string column
    DT = dt.Frame(A=["dfv", None, None, "adfknlkad", None])
    RES = DT[:, f.A == None]
    assert RES.to_list()[0] == [0, 1, 1, 0, 1]

#-------------------------------------------------------------------------------
# split_into_nhot
#-------------------------------------------------------------------------------

def test_split_into_nhot_noarg():
    with pytest.raises(ValueError) as e:
        noop(dt.str.split_into_nhot())
    assert ("Required parameter frame is missing" == str(e.value))

def test_split_into_nhot_none():
    f0 = dt.str.split_into_nhot(None)
    assert f0 is None

def test_split_into_nhot_empty():
    f0 = dt.str.split_into_nhot(dt.Frame(["", None]))
    assert_equals(f0, dt.Frame())

@pytest.mark.parametrize('sort', [False, True])
def test_split_into_nhot0(sort):
    f0 = dt.Frame(["cat, dog, mouse, peacock, frog",
                   "armadillo, fox, hedgehog",
                   None,
                   "dog, fox, mouse, cat, peacock",
                   "horse, raccoon, cat, frog, dog"])
    f1 = dt.str.split_into_nhot(f0, sort = sort)
    frame_integrity_check(f1)
    fr = dt.Frame({"cat":       [1, 0, None, 1, 1],
                   "dog":       [1, 0, None, 1, 1],
                   "mouse":     [1, 0, None, 1, 0],
                   "peacock":   [1, 0, None, 1, 0],
                   "frog":      [1, 0, None, 0, 1],
                   "armadillo": [0, 1, None, 0, 0],
                   "fox":       [0, 1, None, 1, 0],
                   "hedgehog":  [0, 1, None, 0, 0],
                   "horse":     [0, 0, None, 0, 1],
                   "raccoon":   [0, 0, None, 0, 1]})
    assert set(f1.names) == set(fr.names)
    if sort :
        assert list(f1.names) == sorted(fr.names)
    fr = fr[:, f1.names]
    assert f1.names == fr.names
    assert f1.stypes == (dt.stype.bool8, ) * f1.ncols
    assert f1.shape == fr.shape
    assert f1.to_list() == fr.to_list()

def test_split_into_nhot1():
    f0 = dt.Frame(["  meow  \n",
                   None,
                   "[ meow]",
                   "['meow' ,purr]",
                   '(\t"meow", \'purr\')',
                   "{purr}"])
    f1 = dt.str.split_into_nhot(f0)
    frame_integrity_check(f1)
    fr = dt.Frame(meow=[1, None, 1, 1, 1, 0], purr=[0, None, 0, 1, 1, 1])
    assert set(f1.names) == set(fr.names)
    fr = fr[..., f1.names]
    assert f1.shape == fr.shape == (6, 2)
    assert f1.stypes == (dt.stype.bool8, dt.stype.bool8)
    assert f1.to_list() == fr.to_list()

def test_split_into_nhot_sep():
    f0 = dt.Frame(["a|b|c", "b|a", None, "a|c"])
    f1 = dt.str.split_into_nhot(f0, sep="|")
    assert set(f1.names) == {"a", "b", "c"}
    fr = dt.Frame(a=[1, 1, None, 1], b=[1, 1, None, 0], c=[1, 0, None, 1])
    assert set(f1.names) == set(fr.names)
    assert f1.to_list() == fr[:, f1.names].to_list()

def test_split_into_nhot_quotes():
    f0 = dt.str.split_into_nhot(dt.Frame(['foo, "bar, baz"']))
    f1 = dt.str.split_into_nhot(dt.Frame(['foo, "bar, baz']))
    assert set(f0.names) == {"foo", "bar, baz"}
    assert set(f1.names) == {"foo", '"bar', "baz"}

def test_split_into_nhot_bad():
    f0 = dt.Frame([[1.25], ["foo"], ["bar"]])
    with pytest.raises(ValueError) as e:
        dt.str.split_into_nhot(f0)
    assert ("Function split_into_nhot() may only be applied to a single-column "
            "Frame of type string; got frame with 3 columns" == str(e.value))

    with pytest.raises(TypeError) as e:
        dt.str.split_into_nhot(f0[:, 0])
    assert ("Function split_into_nhot() may only be applied to a single-column "
            "Frame of type string; received a column of type float64" ==
            str(e.value))

    with pytest.raises(ValueError) as e:
        dt.str.split_into_nhot(f0[:, 1], sep=",;-")
    assert ("Parameter sep in split_into_nhot() must be a single character"
            in str(e.value))

@pytest.mark.parametrize("seed, st", [(random.getrandbits(32), stype.str32),
                                      (random.getrandbits(32), stype.str64)])
def test_split_into_nhot_long(seed, st):
    random.seed(seed)
    n = int(random.expovariate(0.0001) + 100)
    col1 = [random.getrandbits(1) for _ in range(n)]
    col2 = [random.getrandbits(1) for _ in range(n)]
    col3 = [random.getrandbits(1) for _ in range(n)]
    col4 = [0] * n

    data = [",".join(["liberty"] * col1[i] +
                     ["equality"] * col2[i] +
                     ["justice"] * col3[i]) for i in range(n)]

    # Introduce 1% of None's, making sure we preserve data at freedom_index
    na_indices = random.sample(range(n), n // 100)
    freedom_index = random.randint(0, n - 1)
    for i in na_indices:
        if i == freedom_index: continue
        col1[i] = None
        col2[i] = None
        col3[i] = None
        col4[i] = None
        data[i] = None
    col4[freedom_index] = 1
    data[freedom_index] += ", freedom"

    f0 = dt.Frame(data, stype=st)
    assert f0.stypes == (st,)
    assert f0.shape == (n, 1)
    f1 = dt.str.split_into_nhot(f0)
    assert f1.shape == (n, 4)
    fr = dt.Frame(liberty=col1, equality=col2, justice=col3, freedom=col4)
    assert set(f1.names) == set(fr.names)
    f1 = f1[..., fr.names]
    assert f1.to_list() == fr.to_list()

def test_split_into_nhot_view():
    f0 = dt.Frame(A=["cat,dog,mouse", "mouse", None, "dog, cat"])
    f1 = dt.str.split_into_nhot(f0[::-1, :])
    f2 = dt.str.split_into_nhot(f0[3, :])
    assert set(f1.names) == {"cat", "dog", "mouse"}
    assert f1[:, ["cat", "dog", "mouse"]].to_list() == \
           [[1, None, 0, 1], [1, None, 0, 1], [0, None, 1, 1]]
    assert set(f2.names) == {"cat", "dog"}
    assert f2[:, ["cat", "dog"]].to_list() == [[1], [1]]

def test_split_into_nhot_deprecated():
    DT = dt.Frame(["a, b, c"])
    with pytest.warns(FutureWarning):
        RES = dt.split_into_nhot(DT)
    EXP = dt.Frame([[1], [1], [1]], names=["a", "b", "c"], stype=dt.bool8)
    assert_equals(RES, EXP)


import hashlib
import zlib
from PyQt5 import QtCore, QtGui, QtWidgets
from file_browse import *

data=[
        {'Check':'','More':'','Details':'','@timestamp':'2023/03/12 12:00:00','Rule':'Enumeration of users or Groups','Severity':'low','Risk Score':'21','Reason':'process event with process dsmemberutil, parent process bash, by root'},
        {'Check':'','More':'','Details':'','@timestamp':'2023/03/12 12:00:01','Rule':'Potential persitance via login hook','Severity':'medium','Risk Score':'50','Reason':'modify key-value pairs in plist files to influence system behaviors, such as hiding the execution of an application (i.e. Hidden Window) or running additional commands for persistence '},
        {'Check':'','More':'','Details':'','@timestamp':'2023/03/12 12:00:02','Rule':'Malware prevention alert','Severity':'high','Risk Score':'73','Reason':'malware, intrusion_detection file event with process AYHelperService'},
        {'Check':'','More':'','Details':'','@timestamp':'2023/03/12 12:00:00','Rule':'Enumeration of users or Groups','Severity':'low','Risk Score':'21','Reason':'process event with process dsmemberutil, parent process bash, by root'}
]
class Ui_FileScan(QtWidgets.QWidget):
    def __init__(self):
        super().__init__()
        self.vLayout1 = QtWidgets.QVBoxLayout()
        self.vLayout2=QtWidgets.QVBoxLayout()
        self.layout = QtWidgets.QHBoxLayout()
        self.bigLayout=QtWidgets.QVBoxLayout()

        self.add_label("Sandbox",1.5,Qt.AlignCenter,20)

        self.add_file_browser_layer1()
        self.add_submit_url_layer2()

        self.buttonClose = QtWidgets.QPushButton('Back')
        self.buttonClose.clicked.connect(self.close)

        self.vLayout1.addStretch()
        self.vLayout2.addStretch()

        self.bigLayout.addLayout(self.layout)
        self.add_label("Drag your file into left field or click to select a file",0.5,Qt.AlignVCenter,10)

        self.bigLayout.addStretch()

        self.setLayout(self.bigLayout)

    def add_label(self,title,width,alignment,fontsize):
        label_princ = QtWidgets.QLabel(title, self)
        label_princ.setStyleSheet("border: "+str(width)+"px solid black;")
        label_princ.setAlignment(alignment)
        label_princ.setFont(QtGui.QFont('Times',fontsize))
        self.bigLayout.addWidget(label_princ)
    def add_file_browser_layer1(self):
        self.fileBrowser=FileBrowser('Open File')
        self.vLayout1.addWidget(self.fileBrowser)
        self.buttonAnayze = QtWidgets.QPushButton('Analyze')
        self.buttonAnayze.clicked.connect(self.analize_file)
        self.buttonAnayze.setMaximumWidth(200)
        self.vLayout1.addWidget(self.buttonAnayze)
        self.layout.addLayout(self.vLayout1)
    def add_submit_url_layer2(self):
        self.url_label= QtWidgets.QLineEdit( self)
        self.url_label.setPlaceholderText("Submit url or hash")

        self.url_label.setAlignment(Qt.AlignLeft | Qt.AlignHCenter)
        self.url_label.setStyleSheet("border: 1px solid black; background-color: rgb(229, 228, 226);")
        self.url_label.setFixedSize(400,400)

        buttonClose = QtWidgets.QPushButton('Submit')
        buttonClose.clicked.connect(self.action_submit)

        self.vLayout2.addWidget(self.url_label)
        self.vLayout2.addWidget(buttonClose)

        self.layout.addLayout(self.vLayout2)
    def action_submit(self):
        #https://www.travelandleisure.com/thmb/n4LZNPWDaJnGGl4jz988ms4u-Pk=/1500x0/filters:no_upscale():max_bytes(150000):strip_icc()/header-PARISPANDA1221-87f0c4cc46bf423ebcaf669c50912c3c.jpg

        print(self.url_label.text())
    def analize_file(self):
        try:
            file=self.fileBrowser.getPaths()[0]
        except:
            file='./table.py'

        # icon_1=QtWidgets.QLabel()
        # pixmap=QtGui.QPixmap('./icons/done.svg')
        # pixmap= pixmap.scaled(30, 30,Qt.KeepAspectRatio)
        # icon_1.setPixmap(pixmap)
        # icon_1.setAlignment(Qt.AlignLeft)
        # self.vLayout1.addWidget(icon_1)

        label_2 = QtWidgets.QLabel("Summary", self)
        label_2.setStyleSheet("border: 1px solid black;")
        label_2.setAlignment(Qt.AlignLeft | Qt.AlignHCenter)
        self.vLayout1.addWidget(label_2)

        tableWidget=QtWidgets.QTableWidget()
        cols=2
        rows=8
        tableWidget.setColumnCount(cols)
        tableWidget.setRowCount(rows)

        tableWidget.setAlternatingRowColors(True)
        tableWidget.setWindowTitle("File analysis")
        tableWidget.setHorizontalHeaderLabels(data[0].keys())
        [tableWidget.horizontalHeader().setSectionResizeMode(i,QtWidgets.QHeaderView.ResizeToContents) for i in range(cols-1)]
        tableWidget.horizontalHeader().setSectionResizeMode(cols-1,QtWidgets.QHeaderView.Stretch)
        self.vLayout1.addWidget(tableWidget)

        with open(file,'rb') as f:
            content=f.read()
        file_data={"Filename":file,"File type":"binary data","MD5":hashlib.md5(content).hexdigest(),"Sha256":hashlib.sha256(content).hexdigest(),"Sha512":hashlib.sha512(content).hexdigest(),"CRC32":hex(zlib.crc32(content) & 0xffffffff),"Yara":"None Matched","Score":"32"}
        for i, (k, v) in enumerate(file_data.items()):
            tableWidget.setItem(i,0,QtWidgets.QTableWidgetItem(k))
            tableWidget.setItem(i,1,QtWidgets.QTableWidgetItem(v))

        self.buttonAnayze.hide()
        self.fileBrowser.hide()

        self.bigLayout.addWidget(self.buttonClose)


import pygame
import field_and_pointer as fp
import argparser as ap

# Stack, field, pointer and the 2-input operators
stackstack = [[]]
the_field = fp.Field(fp.load_code())
pointer = fp.Pointer((0, 0), (1, 0))
operators = {
    "+": lambda x1, x2: stackstack[-1].append(x1 + x2),
    "-": lambda x1, x2: stackstack[-1].append(x2 - x1),
    "*": lambda x1, x2: stackstack[-1].append(x1 * x2),
    "/": lambda x1, x2:
    stackstack[-1].append(int(float(x2) // float(x1)))
    if x1 else stackstack[-1].append(0),
    "%": lambda x1, x2: stackstack[-1].append(x2 % x1),
    "`": lambda x1, x2:
    stackstack[-1].append(1) if x2 > x1 else stackstack[-1].append(0),
    "\\": lambda x1, x2: stackstack[-1].extend([x1, x2]),
    "g": lambda x1, x2: stackstack[-1].append(ord(the_field.get_char(x2, x1)))}

# Global constants related to pygame
CHAR_WIDTH = 12
CHAR_HEIGHT = 28
SCREEN_HEIGHT_MODIFIER = 300
SCREEN_HEIGHT = the_field.Y * CHAR_HEIGHT + SCREEN_HEIGHT_MODIFIER
SCREEN_WIDTH = the_field.X * CHAR_WIDTH + 500
BG_COLOR = (52, 52, 52)
STACK_BG_COLOR = (0, 0, 0, 100)
STACK_OUTPUT_COLOR = (230, 200, 70)
SOSS_OUTPUT_COLOR = (70, 230, 200)
POINTER_COLOR = (255, 255, 255, 130)
STACK_CHAR_HEIGHT = 16
STACK_CHAR_WIDTH = 10
ARGS = ap.parse_arguments()
_paused = False
_step_once = False
_reset = False

# Pygame surface inits
if not ARGS.OUTPUT_MODE:
    screen = pygame.display.set_mode((SCREEN_WIDTH, SCREEN_HEIGHT))
    background = pygame.Surface(screen.get_size()).convert()
    pointer_rect = pygame.Surface((CHAR_WIDTH, CHAR_HEIGHT), pygame.SRCALPHA)
    pointer_rect.fill(POINTER_COLOR)
    stacksurf = pygame.Surface((SCREEN_WIDTH, SCREEN_HEIGHT_MODIFIER),
                               pygame.SRCALPHA)
    outsurf = pygame.Surface((int(float(SCREEN_WIDTH) / 2.0),
                             SCREEN_HEIGHT_MODIFIER),
                             pygame.SRCALPHA)
    pygame.display.set_caption("Befunge-98 Interpreter")
    # Pygame font inits
    pygame.font.init()
    stackfont = pygame.font.Font("../font/Inconsolata.otf", 18)
    codefont = pygame.font.Font("../font/Inconsolata.otf", 24)

# Global constants related to bf98.py
_outcount = 0
_outline = 0
_instring = ""


from flask import Flask, request

app = Flask(__name__)

@app.route('/02-server')
def server01_views():
    return "This is my first response by Ajax"

@app.route('/03-server')
def server03_views():
    uname = request.args['uname']
    return "欢迎" + uname

if __name__ == '__main__':
    app.run(debug=True)


import datetime
import json
import os
import random
import string
import sys
import time
import urllib.request

os.system("title WARP-PLUS-CLOUDFLARE By Tran Thai Tuan Anh")
os.system('cls' if os.name == 'nt' else 'clear')
print ("[+] About script: With this script, you can getting unlimited GB on Warp+")
print ("[+] This script coded by Tran Thai Tuan Anh")
print ("----------")
referrer = input("[#] Enter your WARP+ ID:")
def genString(stringLength):
	try:
		letters = string.ascii_letters + string.digits
		return ''.join(random.choice(letters) for i in range(stringLength))
	except Exception as error:
		print(error)
def digitString(stringLength):
	try:
		digit = string.digits
		return ''.join((random.choice(digit) for i in range(stringLength)))
	except Exception as error:
		print(error)
url = f'https://api.cloudflareclient.com/v0a{digitString(3)}/reg'
def run():
	try:
		install_id = genString(22)
		body = {"key": "{}=".format(genString(43)),
				"install_id": install_id,
				"fcm_token": "{}:APA91b{}".format(install_id, genString(134)),
				"referrer": referrer,
				"warp_enabled": False,
				"tos": datetime.datetime.now().isoformat()[:-3] + "+02:00",
				"type": "Android",
				"locale": "es_ES"}
		data = json.dumps(body).encode('utf8')
		headers = {'Content-Type': 'application/json; charset=UTF-8',
					'Host': 'api.cloudflareclient.com',
					'Connection': 'Keep-Alive',
					'Accept-Encoding': 'gzip',
					'User-Agent': 'okhttp/3.12.1'
					}
		req         = urllib.request.Request(url, data, headers)
		response    = urllib.request.urlopen(req)
		status_code = response.getcode()
		return status_code
	except Exception as error:
		print(error)

good = 0
bad = 0
while True:
	result = run()
	if result == 200:
		good += 1
		os.system('cls' if os.name == 'nt' else 'clear')
		print("WARP-PLUS-CLOUDFLARE (script) by Tran Thai Tuan Anh")
		print("")
		animation = ["10%","20%", "30%", "40%", "50%", "60%", "70%", "80%", "90%", "100%"]
		for i in range(len(animation)):
			time.sleep(0.5)
			sys.stdout.write("\rprocessing..." + animation[i % len(animation)])
			sys.stdout.flush()
		os.system('cls' if os.name == 'nt' else 'clear')
		print("WARP-PLUS-CLOUDFLARE (script) by Tran Thai Tuan Anh")
		print(f"\n[-] ID: {referrer}")
		print(f"[-] {good} GB has been successfully added to your account.")
		print(f"[-] Total: {good} Good {bad} Bad")
		print("[-] After 18 seconds, a new request will be sent.")
		time.sleep(18)
	else:
		bad += 1
		os.system('cls' if os.name == 'nt' else 'clear')
		print("WARP-PLUS-CLOUDFLARE (script) by Tran Thai Tuan Anh")
		print("")
		print("[-] Error when connecting to server.")
		print(f"[-] Total: {good} Good {bad} Bad")
		print("[-] After 18 seconds, a new request will be sent.")
		time.sleep(18)


# This defines the model
import torch.nn as nn

from torchvision import models

import torch.nn as nn
import torch

class MyEnsemble(nn.Module):

    def __init__(self, modelA, modelB, input):
        super(MyEnsemble, self).__init__()
        self.modelA = modelA
        self.modelB = modelB
        self.fc1 = nn.Linear(input, 2)

    def forward(self, x):
        out1 = self.modelA(x)
        out2 = self.modelB(x)
        out = out1 + out2
        x = self.fc1(out)
        return torch.softmax(x, dim=1)

"""
def build_model( num_classes=2):
    model1 = models.resnet101(pretrained=True)
    num_ftrs1 = model1.fc.in_features
    model1.fc = nn.Linear(num_ftrs1, num_classes)

    model2 = models.resnet152(pretrained=True)
    num_ftrs2 = model2.fc.in_features
    model2.fc = nn.Linear(num_ftrs2, num_classes)

    model = MyEnsemble(model1, model2,num_classes)

    return model

"""
def build_model(pretrained=True, fine_tune=True, num_classes=10):
    model=models.resnet152(pretrained=True)
    num_ftrs1 = model.fc.in_features
    model.fc = nn.Linear(num_ftrs1, num_classes)
    return model


from django.shortcuts import render

from .models import AreaOfExpertise, Expert

def browser(request):
    aoe = AreaOfExpertise.objects.all().order_by('field')

    experts = Expert.objects.all()

    institutes = set()
    for expert in experts:
        institutes.add(expert.institute) if expert.institute is not None else None

    name = request.GET.get('name')
    institute = request.GET.get('institute')
    expertise = request.GET.get('expertise')

    if name:
        experts = experts.filter(name__contains=request.GET.get('name'))

    if institute:
        experts = experts.filter(institute=request.GET.get('institute'))

    if expertise:
        experts = experts.filter(expertise=AreaOfExpertise.objects.filter(field=expertise)).distinct()

    return render(request, 'expertBrowser.html', {'experts': experts, 'institutes': institutes, 'aoe': aoe, 'name': name, 'institute': institute, 'expertise': expertise})

def viewer(request, id):
    expert = Expert.objects.get(id=id)
    return render(request, 'expertViewer.html', {'expert': expert})


import serial
import numpy as np
import matplotlib.pyplot as plt

# シリアルポートの設定（必要に応じて変更）
ser = serial.Serial('/dev/ttyACM0', 115200)

# グラフの初期化
fig, ax = plt.subplots()
x_data, y_data = [], []
line, = ax.plot(x_data, y_data, 'o', markersize=6)

# 3秒間のデータを保持するためのリングバッファ
max_data_points = 300  # 1秒あたり100データ点 × 3秒
x_ring_buffer = [0] * max_data_points
y_ring_buffer = [0] * max_data_points
current_index = 0

def update_plot(x, y):
    # データをプロット
    x_data.append(x)
    y_data.append(y)
    line.set_data(x_data, y_data)

    # データが3秒分を超えた場合、古いデータを削除
    if len(x_data) > max_data_points:
        x_data.pop(0)
        y_data.pop(0)

    # グラフを更新
    plt.pause(0.01)

while True:
    data = ser.readline().decode().strip()
    if data.startswith("(") and data.endswith(")"):
        data = data[1:-1]
        x, y, _ = map(int, data.split(", "))

        # リアルタイムプロットを更新
        update_plot(x, y)


from django import http
from django.http.response import Http404, HttpResponseForbidden
from rest_framework import viewsets, mixins, status
from rest_framework.response import Response
from rest_framework.decorators import action, api_view, permission_classes

from django.shortcuts import get_object_or_404

from .models import Answer, Enrollment, Offering, User, Exercise, UserAnswerSummary
from .serializers import AnswerSerializer, OfferingSerializer, UserAnswerSummarySerializer, UserSerializer, ExerciseSerializer
from .permissions import IsAdminOrSelf, IsAdminUser, IsEnrolledInOfferingOrIsStaff

class UserViewSet(viewsets.ModelViewSet):
    queryset = User.objects.all()
    serializer_class = UserSerializer
    permission_classes = [IsAdminOrSelf]

class OfferingViewSet(viewsets.ReadOnlyModelViewSet):
    queryset = Offering.objects.all()
    serializer_class = OfferingSerializer
    permission_classes = [IsAdminUser]

@api_view(['GET'])
@permission_classes([IsAdminUser])
def get_enrolled_students(request, off_pk):
    students = User.objects.filter(enrollment__offering_id=off_pk)
    return Response(UserSerializer(students, many=True).data)

class ExerciseViewSet(mixins.CreateModelMixin, mixins.ListModelMixin, viewsets.GenericViewSet):
    serializer_class = ExerciseSerializer

    def get_permissions(self):
        permissions = [IsAdminUser]
        if self.action == 'list':
            permissions = [IsEnrolledInOfferingOrIsStaff]

        return [permission() for permission in permissions]

    def create(self, request, off_pk=None):
        offering = get_object_or_404(Offering, pk=off_pk)

        exercise, new = offering.exercise_set.update_or_create(slug=request.data['slug'],
            defaults={
                'url': request.data['url'],
                'type': request.data['type'],
                'topic': request.data['topic'],
                'group': request.data['group'],
            })

        s = ExerciseSerializer(exercise)
        return Response(s.data, status=status.HTTP_201_CREATED)

    def list(self, request, off_pk=None):
        offering = get_object_or_404(Offering, pk=off_pk)
        exercises_json = ExerciseSerializer(offering.exercise_set.all(), many=True)
        return Response(exercises_json.data)

class AnswerViewSet(viewsets.ModelViewSet):
    serializer_class = AnswerSerializer
    permission_classes = [IsEnrolledInOfferingOrIsStaff]

    def get_queryset(self):
        offering_pk = self.kwargs.get('off_pk')
        get_object_or_404(Offering, pk=offering_pk)
        exercise = get_object_or_404(Exercise, offering=offering_pk, slug=self.kwargs.get('ex_slug'))

        f = user_filter(self.request)

        return Answer.objects.filter(exercise=exercise, **f)

    def create(self, request, off_pk=None, ex_slug=None):
        get_object_or_404(Offering, pk=off_pk)
        exercise = get_object_or_404(Exercise, slug=ex_slug, offering=off_pk)
        if not exercise.allow_submissions:
            return HttpResponseForbidden()

        answer_data = {
            'user': request.user.pk,
            'exercise': exercise.pk,
            'test_results': request.data['test_results'],
            'student_input': request.data['student_input'],
            'points': request.data['points']
        }

        answer = AnswerSerializer(data=answer_data)
        if answer.is_valid():
            answer.save()
            return Response(answer.data, status=status.HTTP_201_CREATED)

        return Response(answer.errors, status=status.HTTP_400_BAD_REQUEST)

    def list_answers_by_student(self, request, off_pk, ex_slug, student_pk):
        get_object_or_404(Offering, pk=off_pk)
        exercise = get_object_or_404(Exercise, offering=off_pk, slug=ex_slug)

        filters = {
            'exercise': exercise
        }

        if self.request.user.is_staff:
            filters['user'] = self.kwargs.get('student_pk')
        elif self.request.user.pk != int(student_pk):
            return HttpResponseForbidden()

        answers = Answer.objects.filter(**filters)
        answers_json = AnswerSerializer(answers, many=True)

        return Response(answers_json.data, status=status.HTTP_200_OK)

def user_filter(request):
    user_pk = request.user.pk
    request_user_pk = request.GET.get('user')
    if request_user_pk and request_user_pk != user_pk and request.user.is_staff:
        user_pk = request_user_pk
    filters = {}
    if not request.user.is_staff or request_user_pk:
        # Can only see own summaries if not admin
        filters['user__pk'] = user_pk
    return filters

@api_view(['GET'])
@permission_classes([IsEnrolledInOfferingOrIsStaff])
def get_latest_answer_by_student(request, off_pk, ex_slug, student_pk):
    offering = get_object_or_404(Offering, pk=off_pk)
    try:
        answer = Answer.objects.filter(
            user__pk=student_pk,
            exercise__slug=ex_slug,
            exercise__offering=offering,
        ).latest('pk')
    except Answer.DoesNotExist:
        raise Http404
    return Response(AnswerSerializer(answer).data)

@api_view(['GET'])
@permission_classes([IsEnrolledInOfferingOrIsStaff])
def get_answer(request, off_pk, ex_slug, ans_pk):
    get_object_or_404(Offering, pk=off_pk)
    answer = get_object_or_404(Answer, pk=ans_pk, **user_filter(request))
    return Response(AnswerSerializer(answer).data)

@api_view(['GET'])
@permission_classes([IsEnrolledInOfferingOrIsStaff])
def get_previous_answer(request, off_pk, ex_slug, ans_pk):
    get_object_or_404(Offering, pk=off_pk)

    filters = user_filter(request)
    try:
        answer = Answer.objects.filter(
            exercise__slug=ex_slug,
            pk__lt=ans_pk,
            **filters).latest('pk')
    except Answer.DoesNotExist:
        raise Http404()
    return Response(AnswerSerializer(answer).data)

@api_view(['GET'])
@permission_classes([IsEnrolledInOfferingOrIsStaff])
def get_next_answer(request, off_pk, ex_slug, ans_pk):
    get_object_or_404(Offering, pk=off_pk)

    filters = user_filter(request)
    try:
        answer = Answer.objects.filter(
            exercise__slug=ex_slug,
            pk__gt=ans_pk,
            **filters).earliest('pk')
    except Answer.DoesNotExist:
        raise Http404()
    return Response(AnswerSerializer(answer).data)

@api_view(['GET'])
@permission_classes([IsEnrolledInOfferingOrIsStaff])
def list_summaries(request, off_pk):
    offering = get_object_or_404(Offering, pk=off_pk)

    filters = user_filter(request)
    all_summaries = UserAnswerSummary.objects.filter(**filters).filter(exercise__offering=offering).prefetch_related('exercise')
    all_summaries_json = UserAnswerSummarySerializer(all_summaries, many=True)

    return Response(all_summaries_json.data, status=status.HTTP_200_OK)

@api_view(['GET'])
@permission_classes([IsEnrolledInOfferingOrIsStaff])
def list_summaries_for_exercise(request, off_pk, ex_slug):
    get_object_or_404(Offering, pk=off_pk)
    exercise = get_object_or_404(Exercise, offering=off_pk, slug=ex_slug)

    filters = user_filter(request)
    filters['exercise'] = exercise
    all_summaries = UserAnswerSummary.objects.filter(**filters).prefetch_related('exercise')
    all_summaries_json = UserAnswerSummarySerializer(all_summaries, many=True)

    return Response(all_summaries_json.data, status=status.HTTP_200_OK)

@api_view(['GET'])
@permission_classes([IsAdminUser])
def list_students_that_tried_exercise(request, off_pk, ex_slug):
    get_object_or_404(Offering, pk=off_pk)
    exercise = get_object_or_404(Exercise, offering=off_pk, slug=ex_slug)

    all_students = User.objects.filter(answer__exercise__slug=ex_slug).distinct()
    all_students_json = UserSerializer(all_students, many=True)

    return Response(all_students_json.data, status=status.HTTP_200_OK)

@api_view(['GET'])
@permission_classes([IsAdminUser])
def activate_exercise(request, off_pk, ex_slug):
    get_object_or_404(Offering, pk=off_pk)
    exercise = get_object_or_404(Exercise, offering=off_pk, slug=ex_slug)
    exercise.allow_submissions = True
    exercise.save()
    return Response('OK', status=status.HTTP_200_OK)

@api_view(['GET'])
@permission_classes([IsAdminUser])
def deactivate_exercise(request, off_pk, ex_slug):
    get_object_or_404(Offering, pk=off_pk)
    exercise = get_object_or_404(Exercise, offering=off_pk, slug=ex_slug)
    exercise.allow_submissions = False
    exercise.save()
    return Response('OK', status=status.HTTP_200_OK)


# -*- coding: utf-8 -*-
import requests
import os

# 6000 is a large number to make sure we get all the components of a collection. Please do note that RISE also has a pagination feature,
# which can be implemented by clients if they wish.
per_page = 6000

# getting the list of collections that the user has access to:
collections_response = requests.get(f'https://rise.mpiwg-berlin.mpg.de/api/collections?per_page={per_page}')

collections = collections_response.json()
# each accessible collections has a name, a uuid, and a number of resources.
# print(collections)
idx = 1
for collection in collections:
      print(f'collection at index: {idx}')
      idx += 1
      print(collection)

# picking a collection by its index
# collection_index = 1
# collection = collections[collection_index]

results = list(filter(lambda collection: collection['name'] == 'MPIWG - 哈佛燕京圖書館藏珍稀方志', collections))
collection = results[0]
print(collection['uuid'])
collection_uuid = collection['uuid']

# we grab all resources for this collection
resources_response = requests.get(f'https://rise.mpiwg-berlin.mpg.de/api/collections/{collection_uuid}/resources?per_page={per_page}')

corpus_path = './corpus'
if not os.path.exists(corpus_path):
      os.makedirs(corpus_path)

for resource in resources_response.json():
      uuid = resource['uuid']
      resource_name = resource['name']
      print(resource_name)

      if not os.path.exists(corpus_path + "/" + resource_name):
            os.makedirs(corpus_path + "/" + resource_name)
      sections = requests.get("https://rise.mpiwg-berlin.mpg.de/api/resources/"+ resource['uuid'] +"/sections")

      for section in sections.json():
            print(section)
            print(section['uuid'])
            section_name = section['name']
            section_path = corpus_path + "/" + resource_name + "/" + section_name
            file = open(section_path +".txt", "w")
            content_units = requests.get("https://rise.mpiwg-berlin.mpg.de/api/sections/"+ section['uuid'] +"/content_units?per_page=6000")
            for content_unit in content_units.json():
                  print(content_unit)
                  file.write(content_unit['content'])
            file.close()


from fastapi import HTTPException, status
from odmantic.bson import ObjectId
from typing import List

# Import helpers
from helpers.database import db

# Import Models
from models.currency import Currency

async def currency_exists(cid: ObjectId, loc=None):
    if loc is None:
        loc = []
    loc = ['body'] + loc + ['currency']

    currency = await db.find_one(Currency, Currency.id == cid)
    if currency is None:
        raise HTTPException(
            status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
            detail=[
                dict(
                    loc=loc,
                    msg='not exists',
                    type='value_error.not.exists'
                )
            ]
        )

    return currency

async def currencies_exists(cids: List[ObjectId], loc=None):
    if loc is None:
        loc = []
    loc = ['body'] + loc

    errors = list()
    if isinstance(cids, List):
        for i, cid in enumerate(cids):
            country = await db.find_one(Currency, Currency.id == cid)
            if country is None:
                errors.append(
                    dict(
                        loc=loc + [i, 'currency'],
                        msg='not exists',
                        type='value_error.not.exists'
                    )
                )

    if len(errors) > 0:
        raise HTTPException(
            status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
            detail=errors
        )


from os import name
from django.urls import path
#from .views import HomePageView
from . import views
urlpatterns = [
	path('',views.teacher_index,name='teacher'),  # Homepag

    path('<slug:teacher_id>/<int:choice>/Classes/', views.teacher_home, name="teacher_home"), #Student Batches
    path('<slug:classid>/Students', views.view_teacher_students, name='teacher_stud'),
    path('info/<slug:student>/', views.each_student_info, name='each_student_info'),
    path('<slug:stud>/<slug:course>/<slug:teach>/', views.view_student_attendence, name='student_attendence'),

    # Assignment Routes

    path('<slug:class_id>/<slug:course>/', views.view_assignments_page, name='view_assignments_page'),
    path('/add/<slug:class_id>/<slug:course>/added/',views.add_assigment,name='add_assigment'),

    # get_assigment is used to view teacter's submitted Assignments and Delete is for to Delete Their Assignments
    path('get_assignments_data/', views.get_submitted_assignments, name='view_assignments'),
    path('delete_assignmet/<slug:ta>/deleted', views.delete_assignments, name='delete_assignments'),

    path('view_own_attendance',views.view_attendance,name='view_attendance'),

    path('promote/', views.promote_students, name='promote_students'),
    path('msgfromhod/',views.messages_from_hod, name='messages_from_hod'),
    # Marks Routes
    path('marks/<slug:stud>/<slug:course>/<slug:teach>/', views.teacher_view_marks, name='teacher_view_marks'),
    path('<slug:stud>/<slug:course>/<slug:teach>/take', views.take_marks, name='take_marks'),

    path('profile', views.teacher_profile, name='teacher_profiles'),

    path('profile/<slug:teacher>/update', views.update_profile, name='update_profile'),
    # path('teacher/<int:id>/teacher_view_students/',views.teacher_view_students, name='teacher_view_students'),
    path('logout/',views.logOut, name='logOut'),

]


from extract_words import extract_words
import gi
gi.require_version('Gtk', '3.0')
gi.require_version('WebKit', '3.0')
from gi.repository import Gtk, Gdk, WebKit

class MainWindow(Gtk.Window):
    def __init__(self):

        Gtk.Window.__init__(self, title='My Window Title')
        self.AUTHEN_CODE = None  # authentication code for API of Shanbay.com
        self.set_title('English Reading Companion')
        self.set_default_size(800, 600)
        self.set_border_width(10)
        self.connect('delete-event', self.quit)
        self.connect('key-press-event', self.on_key_click)

        # Left List
        self.wl_store = Gtk.ListStore(str)
        self.wl_treeview = Gtk.TreeView(model=self.wl_store)
        self.wl_column = Gtk.TreeViewColumn('New [%d]' % len(self.wl_store), Gtk.CellRendererText(), text=0)
        self.wl_treeview.append_column(self.wl_column)
        self.wl_treeview.get_selection().set_mode(Gtk.SelectionMode.MULTIPLE)
        self.wl_treeview.override_background_color(Gtk.StateType.NORMAL,
                                                   Gdk.RGBA.from_color(Gdk.color_parse('CadetBlue3')))

        wl_scrolled_window = Gtk.ScrolledWindow()
        wl_scrolled_window.set_policy(
            Gtk.PolicyType.NEVER, Gtk.PolicyType.AUTOMATIC)
        wl_scrolled_window.add(self.wl_treeview)

        # Middle List
        self.wm_store = Gtk.ListStore(str)
        try:
            with open('undetermined_words.dat', 'r') as f:
                for line in f:
                    self.wm_store.append([line.strip()])
        except FileNotFoundError:
            pass
        self.wm_treeview = Gtk.TreeView(model=self.wm_store)
        self.wm_column = Gtk.TreeViewColumn('Undertermined [%d]' % len(self.wm_store), Gtk.CellRendererText(), text=0)
        self.wm_treeview.append_column(self.wm_column)
        self.wm_treeview.get_selection().set_mode(Gtk.SelectionMode.MULTIPLE)
        self.wm_treeview.override_background_color(Gtk.StateType.NORMAL,
                                                   Gdk.RGBA.from_color(Gdk.color_parse('CadetBlue3')))

        wm_scrolled_window = Gtk.ScrolledWindow()
        wm_scrolled_window.set_policy(
            Gtk.PolicyType.NEVER, Gtk.PolicyType.AUTOMATIC)
        wm_scrolled_window.add(self.wm_treeview)

        # Right List
        self.wr_store = Gtk.ListStore(str)
        self.wr_treeview = Gtk.TreeView(model=self.wr_store)
        self.wr_column = Gtk.TreeViewColumn('Mastered [%d]' % len(self.wr_store), Gtk.CellRendererText(), text=0)
        self.wr_treeview.append_column(self.wr_column)
        self.wr_treeview.get_selection().set_mode(Gtk.SelectionMode.MULTIPLE)
        self.wr_treeview.override_background_color(Gtk.StateType.NORMAL,
                                                   Gdk.RGBA.from_color(Gdk.color_parse('CadetBlue3')))

        wr_scrolled_window = Gtk.ScrolledWindow()
        wr_scrolled_window.set_policy(
            Gtk.PolicyType.NEVER, Gtk.PolicyType.AUTOMATIC)
        wr_scrolled_window.add(self.wr_treeview)

        box_outer = Gtk.Box(orientation=Gtk.Orientation.HORIZONTAL, spacing=10)
        # Left Part
        l_box = Gtk.Box(orientation=Gtk.Orientation.VERTICAL, spacing=0)

        push_btn = Gtk.Button(label='Push To Shanbay.com')
        push_btn.connect('clicked', self.push_to_shanbay)
        l_box.pack_start(wl_scrolled_window, True, True, 0)
        l_box.pack_start(push_btn, False, False, 0)

        box_outer.pack_start(l_box, True, True, 0)

        # Middle Part
        m_box = Gtk.Box(orientation=Gtk.Orientation.VERTICAL, spacing=0)

        m_btn_box = Gtk.Box(orientation=Gtk.Orientation.HORIZONTAL, spacing=0)

        left_btn = Gtk.Button()
        left_btn.add(Gtk.Arrow(Gtk.ArrowType.LEFT, Gtk.ShadowType.NONE))
        left_btn.connect('clicked', self.word_move_left)

        right_btn = Gtk.Button()
        right_btn.add(Gtk.Arrow(Gtk.ArrowType.RIGHT, Gtk.ShadowType.NONE))
        right_btn.connect('clicked', self.word_move_right)

        m_btn_box.pack_start(left_btn, True, True, 0)
        m_btn_box.pack_start(right_btn, True, True, 0)

        import_btn = Gtk.Button(label='Import Words')
        import_btn.connect('clicked', self.import_words)
        m_box.pack_start(wm_scrolled_window, True, True, 0)
        m_box.pack_start(m_btn_box, False, False, 0)
        m_box.pack_start(import_btn, False, False, 0)

        box_outer.pack_start(m_box, True, True, 0)

        # Right Part
        r_box = Gtk.Box(orientation=Gtk.Orientation.VERTICAL, spacing=0)
        save_btn = Gtk.Button(label='Save To File & Clear')
        save_btn.connect('clicked', self.save_to_file)
        r_box.pack_start(wr_scrolled_window, True, True, 0)
        r_box.pack_start(save_btn, False, False, 0)
        box_outer.pack_start(r_box, True, True, 0)

        self.add(box_outer)
        self.show_all()

    def word_move_left(self, *args):
        selection = self.wm_treeview.get_selection()
        model, paths = selection.get_selected_rows()

        for path in paths:
            self.wl_store.insert(0, model[path][:])
        for path in paths[::-1]:
            model.remove(model.get_iter(path))
        self.wl_treeview.set_cursor(Gtk.TreePath(0))
        self.wl_column.set_title('New [%d]' % len(self.wl_store))
        self.wm_column.set_title('Undetermined [%d]' % len(self.wm_store))

    def word_move_right(self, *args):
        selection = self.wm_treeview.get_selection()
        model, paths = selection.get_selected_rows()
        for path in paths:
            self.wr_store.insert(0, model[path][:])
        for path in paths[::-1]:
            model.remove(model.get_iter(path))
        self.wr_treeview.set_cursor(Gtk.TreePath(0))
        self.wr_column.set_title('Mastered [%d]' % len(self.wr_store))
        self.wm_column.set_title('Undetermined [%d]' % len(self.wm_store))

    def move_right_back(self, widget):
        model, paths = self.wr_treeview.get_selection().get_selected_rows()
        for path in paths:
            self.wm_store.insert(0, model[path][:])
        for path in paths[::-1]:
            model.remove(model.get_iter(path))
        self.wm_treeview.set_cursor(Gtk.TreePath(0))
        self.wr_column.set_title('Mastered [%d]' % len(self.wr_store))
        self.wm_column.set_title('Undetermined [%d]' % len(self.wm_store))

    def move_left_back(self, widget):
        model, paths = self.wl_treeview.get_selection().get_selected_rows()
        for path in paths:
            self.wm_store.insert(0, model[path][:])
        for path in paths[::-1]:
            model.remove(model.get_iter(path))
        self.wm_treeview.set_cursor(Gtk.TreePath(0))
        self.wl_column.set_title('New [%d]' % len(self.wl_store))
        self.wm_column.set_title('Undetermined [%d]' % len(self.wm_store))

    def save_to_file(self, widget):
        words_to_save = [row[:][0] for row in self.wr_store]
        if len(words_to_save) > 0:
            with open('known_words.dat', 'a') as f:
                f.write('\n'.join(words_to_save) + '\n')
        self.wr_store.clear()
        self.wr_column.set_title('Mastered [%d]' % len(self.wr_store))
        self.wm_column.set_title('Undetermined [%d]' % len(self.wm_store))

    def push_to_shanbay(self, widget):
        if self.AUTHEN_CODE is None:
            self.AUTHEN_CODE = self.authenticate(self)
        if self.AUTHEN_CODE is None:
            return
        words_to_push = [row[:][0] for row in self.wl_store]
        if len(words_to_push) > 0:
            with open('learning_words.dat', 'a') as f:
                f.write('\n'.join(words_to_push) + '\n')
        self.wl_store.clear()
        self.wl_column.set_title('New [%d]' % len(self.wl_store))
        self.wm_column.set_title('Undetermined [%d]' % len(self.wm_store))

    def import_words(self, widget):
        words_in_window = set([row[:][0] for row in self.wl_store] +
                              [row[:][0] for row in self.wm_store] +
                              [row[:][0] for row in self.wr_store])
        dialog = Gtk.FileChooserDialog("Select file to be opened", self,
                                       Gtk.FileChooserAction.OPEN,
                                       (Gtk.STOCK_CANCEL, Gtk.ResponseType.CANCEL,
                                        Gtk.STOCK_OK, Gtk.ResponseType.OK))
        dialog.show()

        response = dialog.run()
        if response == Gtk.ResponseType.OK:
            words = extract_words(dialog.get_filename(), words_in_window)
        elif response == Gtk.ResponseType.CANCEL:
            words = []
        dialog.destroy()
        for word in words:
            self.wm_store.append([word])
        self.wm_column.set_title('Undetermined [%d]' % len(self.wm_store))

    def on_key_click(self, widget, ev):
        ctrl = ev.state & Gdk.ModifierType.CONTROL_MASK

        if ctrl:
            if ev.keyval == Gdk.KEY_Left:
                self.move_right_back(widget)
            elif ev.keyval == Gdk.KEY_Right:
                self.move_left_back(widget)
            else:
                pass
        else:
            if ev.keyval == Gdk.KEY_Left:
                self.word_move_left(widget)
            elif ev.keyval == Gdk.KEY_Right:
                self.word_move_right(widget)
            else:
                pass

    def _javascript_console_message(self, view, message, line, sourceid):
        return True  # True prevents calling original handler

    def authenticate(self, *args, **kwargs):
        dialog = Gtk.Dialog("Authenticate", self, 0,
                            (Gtk.STOCK_CANCEL, Gtk.ResponseType.CANCEL,
                             Gtk.STOCK_OK, Gtk.ResponseType.OK))
        dialog.set_default_size(800, 600)
        box = dialog.get_content_area()
        box.set_homogeneous(False)

        webview = WebKit.WebView()
        webview.connect("console-message", self._javascript_console_message)
        webview.load_uri("https://api.shanbay.com/oauth2/authorize/?cliend_id=05623ce286a180e4e8c1&response_type=code&state=123")

        scrolled_window = Gtk.ScrolledWindow()
        scrolled_window.add(webview)
        scrolled_window.set_size_request(750, 650)
        box.add(scrolled_window)
        dialog.show_all()

        response = dialog.run()
        if response == Gtk.ResponseType.OK:
            uri = webview.get_uri()
        elif response == Gtk.ResponseType.CANCEL:
            uri = None
        dialog.destroy()
        self.AUTHEN_CODE = uri

    def quit(self, *args, **kwargs):
        with open('undetermined_words.dat', 'w') as f:
            f.write('\n'.join([row[:][0] for row in self.wm_store]))
        Gtk.main_quit(self, *args, **kwargs)

win = MainWindow()
Gtk.main()


"""Control module for the Vapourtec R4 heater."""
from __future__ import annotations

from collections import namedtuple
from collections.abc import Iterable

import aioserial
import pint
from loguru import logger

from flowchem import ureg
from flowchem.components.device_info import DeviceInfo
from flowchem.components.technical.temperature import TempRange
from flowchem.devices.flowchem_device import FlowchemDevice
from flowchem.devices.vapourtec.r4_heater_channel_control import R4HeaterChannelControl
from flowchem.utils.exceptions import InvalidConfigurationError
from flowchem.utils.people import dario, jakob, wei_hsin

try:
    # noinspection PyUnresolvedReferences
    from flowchem_vapourtec import VapourtecR4Commands

    HAS_VAPOURTEC_COMMANDS = True
except ImportError:
    HAS_VAPOURTEC_COMMANDS = False

class R4Heater(FlowchemDevice):
    """R4 reactor heater control class."""

    DEFAULT_CONFIG = {
        "timeout": 0.1,
        "baudrate": 19200,
        "parity": aioserial.PARITY_NONE,
        "stopbits": aioserial.STOPBITS_ONE,
        "bytesize": aioserial.EIGHTBITS,
    }

    ChannelStatus = namedtuple("ChannelStatus", "state, temperature")

    def __init__(
        self,
        name: str = "",
        min_temp: float | list[float] = -100,
        max_temp: float | list[float] = 250,
        **config,
    ) -> None:
        super().__init__(name)
        # Set min and max temp for all 4 channels
        if not isinstance(min_temp, Iterable):
            min_temp = [min_temp] * 4
        if not isinstance(max_temp, Iterable):
            max_temp = [max_temp] * 4
        assert len(min_temp) == len(max_temp) == 4
        self._min_t = min_temp * ureg.degreeC
        self._max_t = max_temp * ureg.degreeC

        if not HAS_VAPOURTEC_COMMANDS:
            msg = (
                "You tried to use a Vapourtec device but the relevant commands are missing!"
                "Unfortunately, we cannot publish those as they were provided under NDA."
                "Contact Vapourtec for further assistance."
            )
            raise InvalidConfigurationError(
                msg,
            )

        self.cmd = VapourtecR4Commands()

        # Merge default settings, including serial, with provided ones.
        configuration = R4Heater.DEFAULT_CONFIG | config
        try:
            self._serial = aioserial.AioSerial(**configuration)
        except aioserial.SerialException as ex:
            msg = f"Cannot connect to the R4Heater on the port <{config.get('port')}>"
            raise InvalidConfigurationError(
                msg,
            ) from ex

        self.device_info = DeviceInfo(
            authors=[dario, jakob, wei_hsin],
            manufacturer="Vapourtec",
            model="R4 reactor module",
        )

    async def initialize(self):
        """Ensure connection."""
        self.device_info.version = await self.version()
        logger.info(f"Connected with R4Heater version {self.device_info.version}")
        temp_limits = {
            ch_num: TempRange(
                min=ureg.Quantity(f"{t[0]} °C"),
                max=ureg.Quantity(f"{t[1]} °C"),
            )
            for ch_num, t in enumerate(zip(self._min_t, self._max_t, strict=True))
        }
        reactor_positions = [
            R4HeaterChannelControl(f"reactor{n+1}", self, n, temp_limits[n])
            for n in range(4)
        ]
        self.components.extend(reactor_positions)

    async def _write(self, command: str):
        """Write a command to the pump."""
        cmd = command + "\r\n"
        await self._serial.write_async(cmd.encode("ascii"))
        logger.debug(f"Sent command: {command!r}")

    async def _read_reply(self) -> str:
        """Read the pump reply from serial communication."""
        reply_string = await self._serial.readline_async()
        logger.debug(f"Reply received: {reply_string.decode('ascii').rstrip()}")
        return reply_string.decode("ascii")

    async def write_and_read_reply(self, command: str) -> str:
        """Send a command to the pump, read the replies and return it, optionally parsed."""
        self._serial.reset_input_buffer()
        await self._write(command)
        logger.debug(f"Command {command} sent to R4!")
        response = await self._read_reply()

        if not response:
            msg = "No response received from heating module!"
            raise InvalidConfigurationError(msg)

        logger.debug(f"Reply received: {response}")
        return response.rstrip()

    async def version(self):
        """Get firmware version."""
        return await self.write_and_read_reply(self.cmd.VERSION)

    async def set_temperature(self, channel, temperature: pint.Quantity):
        """Set temperature to channel."""
        cmd = self.cmd.SET_TEMPERATURE.format(
            channel=channel,
            temperature_in_C=round(temperature.m_as("°C")),
        )
        await self.write_and_read_reply(cmd)
        # Set temperature implies channel on
        await self.power_on(channel)
        # Verify it is not unplugged
        status = await self.get_status(channel)
        if status.state == "U":
            logger.error(
                f"TARGET CHANNEL {channel} UNPLUGGED! (Note: numbering starts at 0)",
            )

    async def get_status(self, channel) -> ChannelStatus:
        """Get status from channel."""
        # This command is a bit fragile for unknown reasons.
        failure = 0
        while True:
            try:
                raw_status = await self.write_and_read_reply(
                    self.cmd.GET_STATUS.format(channel=channel),
                )
                return R4Heater.ChannelStatus(raw_status[:1], raw_status[1:])
            except InvalidConfigurationError as ex:
                failure += 1
                # Allows 3 failures cause the R4 is choosy at times...
                if failure > 3:
                    raise ex
                else:
                    continue

    async def get_temperature(self, channel):
        """Get temperature (in Celsius) from channel."""
        state = await self.get_status(channel)
        return None if state.temperature == "281.2" else state.temperature

    async def power_on(self, channel):
        """Turn on channel."""
        await self.write_and_read_reply(self.cmd.POWER_ON.format(channel=channel))

    async def power_off(self, channel):
        """Turn off channel."""
        await self.write_and_read_reply(self.cmd.POWER_OFF.format(channel=channel))

if __name__ == "__main__":
    import asyncio

    r4_device = R4Heater(port="COM1")

    async def main(heat):
        """Test function."""
        await heat.initialize()
        # Get reactors
        r1, r2, r3, r4 = heat.components()

        await r1.set_temperature("30 °C")
        print(f"Temperature is {await r1.get_temperature()}")

    asyncio.run(main(r4_device))