from __future__ import absolute_import, division, print_function
from scitbx.rigid_body.essence import tardy
from scitbx.graph import tardy_tree
from scitbx.graph import test_cases_tardy_pdb
from scitbx.array_family import flex
from scitbx import matrix
from libtbx.utils import null_out, show_times_at_exit
from libtbx.test_utils import approx_equal, is_above_limit
import random
import sys
from six.moves import range
from six.moves import zip

if (1):
  random.seed(0)
  flex.set_random_seed(0)

class tardy_model(tardy.model):

  def d_e_pot_d_q_via_finite_differences(O, eps=1.e-6):
    result = []
    for body in O.bodies:
      gs = []
      j_orig = body.joint
      q_orig = list(j_orig.get_q())
      for iq in range(j_orig.q_size):
        fs = []
        for signed_eps in [eps, -eps]:
          q_eps = list(q_orig)
          q_eps[iq] += signed_eps
          body.joint = j_orig.new_q(q=q_eps)
          O.flag_positions_as_changed()
          fs.append(O.e_pot())
        gs.append((fs[0]-fs[1])/(2*eps))
      body.joint = j_orig
      O.flag_positions_as_changed()
      result.append(matrix.col(gs))
    return result

  def check_d_e_pot_d_q(O, verbose=0):
    qdd_orig = O.qdd_array()
    ana = O.d_e_pot_d_q()
    fin = O.d_e_pot_d_q_via_finite_differences()
    if (verbose):
      for a,f in zip(ana, fin):
        print("fin:", f.elems)
        print("ana:", a.elems)
      print()
    assert approx_equal(ana, fin)
    assert approx_equal(O.qdd_array(), qdd_orig)

class potential_object(object):

  def __init__(O,
        sites,
        wells,
        restraint_edges,
        restraint_edge_weight=1/0.1**2,
        epsilon=1.e-100):
    if (isinstance(wells, flex.vec3_double)):
      wells = [matrix.col(w) for w in wells]
    O.wells = wells
    O.restraints = []
    for edge in restraint_edges:
      s = [matrix.col(sites[i]) for i in edge]
      O.restraints.append((edge, abs(s[0]-s[1]), restraint_edge_weight))
    O.epsilon = epsilon

  def e_pot(O, sites_moved):
    if (isinstance(sites_moved, flex.vec3_double)):
      sites_moved = [matrix.col(s) for s in sites_moved]
    result = 0
    for s, w in zip(sites_moved, O.wells):
      result += (s - w).dot()
    for edge,d_ideal,w in O.restraints:
      s = [sites_moved[i] for i in edge]
      d_model = abs(s[0]-s[1])
      if (d_model < O.epsilon): continue
      delta = d_ideal - d_model
      result += w * delta**2
    return result

  def d_e_pot_d_sites(O, sites_moved):
    if (isinstance(sites_moved, flex.vec3_double)):
      sites_moved = [matrix.col(s) for s in sites_moved]
    result = []
    for s, w in zip(sites_moved, O.wells):
      result.append(2 * (s - w))
    for edge,d_ideal,w in O.restraints:
      s = [sites_moved[i] for i in edge]
      d_model = abs(s[0]-s[1])
      if (d_model < O.epsilon): continue
      delta = d_ideal - d_model
      g0 = -w * 2 * delta / d_model * (s[0] - s[1])
      result[edge[0]] += g0
      result[edge[1]] -= g0
    return result

def exercise_qd_e_kin_scales(tardy_model):
  def slow():
    result = flex.double()
    for body in tardy_model.bodies:
      bj0 = body.joint
      qd0 = body.joint.qd_zero
      qd = list(qd0)
      for iqd in range(len(qd)):
        qd[iqd] = qd0[iqd] + 1
        body.qd = matrix.col(qd)
        qd[iqd] = qd0[iqd]
        tardy_model.flag_velocities_as_changed()
        body.joint = bj0.time_step_position(qd=body.qd, delta_t=1)
        e_kin = tardy_model.e_kin()
        if (e_kin < 1.e-12):
          result.append(1)
        else:
          result.append(1 / e_kin**0.5)
      body.joint = bj0
      body.qd = body.joint.qd_zero
      tardy_model.flag_positions_as_changed()
    assert tardy_model.e_kin() == 0
    assert len(result) == tardy_model.degrees_of_freedom
    return result
  scales_slow = slow()
  scales_fast = tardy_model.qd_e_kin_scales()
  assert approx_equal(scales_fast, scales_slow)

def exercise_random_velocities(tardy_model):
  prev_qd = tardy_model.pack_qd()
  for e_kin_target in [1, 1.3, 0, 13]:
    tardy_model.assign_random_velocities(e_kin_target=e_kin_target)
    assert approx_equal(tardy_model.e_kin(), e_kin_target)
  assert not approx_equal(tardy_model.pack_qd(), prev_qd, out=None)
  tardy_model.unpack_qd(qd_packed=prev_qd)
  assert approx_equal(tardy_model.pack_qd(), prev_qd)

def exercise_accumulate_in_each_tree():
  def check(n_vertices, edge_list, clusters, nosiet, somiet):
    sites = matrix.col_list([(i,i%2,0) for i in range(n_vertices)])
    labels = [str(i) for i in range(n_vertices)]
    masses = [13, 7, 23, 19, 29, 11, 17][:n_vertices]
    assert len(masses) == n_vertices
    tt = tardy_tree.construct(sites=sites, edge_list=edge_list)
    assert len(tt.cluster_manager.fixed_hinges) == 0
    assert tt.cluster_manager.clusters == clusters
    tm = construct_tardy_model(
      labels=labels, sites=sites, masses=masses, tardy_tree=tt)
    assert tm.root_indices() == list(reversed([i for i,n in nosiet]))
    assert tm.number_of_sites_in_each_tree() == nosiet
    assert tm.sum_of_masses_in_each_tree() == somiet
  #
  check(
    n_vertices=5,
    edge_list=[],
    clusters=[[0],[1],[2],[3],[4]],
    nosiet=[(4, 1), (3, 1), (2, 1), (1, 1), (0, 1)],
    somiet=[(4, 29), (3, 19), (2, 23), (1, 7), (0, 13)])
  check(
    n_vertices=5,
    edge_list=[(0,1), (1,2), (3,4)],
    clusters=[[0,1,2], [3,4]],
    nosiet=[(1, 2), (0, 3)],
    somiet=[(1, 48), (0, 43)])
  check(
    n_vertices=5,
    edge_list=[(0,1), (1,2), (2,3), (3,4)],
    clusters=[[0,1,2], [3], [4]],
    nosiet=[(0, 5)],
    somiet=[(0, 91)])
  check(
    n_vertices=7,
    edge_list=[(0,1), (1,2), (2,3), (2,5), (3,4), (5,6)],
    clusters=[[1,2,3,5], [0], [4], [6]],
    nosiet=[(0, 7)],
    somiet=[(0, 119)])

def exercise_near_singular_hinges():
  # similar to scitbx/graph/test_cases_tardy_pdb.py, "collinear" test case
  r"""
    0                6
    | \             /
    |  2---3---4---5
    | /
    1
  """
  x = -0.5*3**0.5
  y = 0.5
  def build_sites(eps):
    return matrix.col_list([
      (x,-y,0),
      (x,y,0),
      (0,0,0),
      (1,0,0),
      (2,0,eps)])
  edge_list = [(0,1),(0,2),(1,2),(2,3),(3,4)]
  sites = build_sites(eps=0)
  for i,j in edge_list:
    assert approx_equal(abs(sites[i]-sites[j]), 1)
  sites = build_sites(eps=1e-5)
  labels = [str(i) for i in range(len(sites))]
  masses = [1] * len(sites)
  tt = tardy_tree.construct(n_vertices=len(sites), edge_list=edge_list)
  tt.build_tree()
  assert tt.cluster_manager.clusters == [[0,1,2,3], [4]]
  def e_kin_1():
    tm = construct_tardy_model(
      labels=labels,
      sites=sites,
      masses=masses,
      tardy_tree=tt)
    rnd = random.Random(0)
    tm.assign_random_velocities(e_kin_target=1, random_gauss=rnd.gauss)
    assert approx_equal(tm.e_kin(), 1, eps=1e-10)
    tm.dynamics_step(delta_t=0.01)
    return tm.e_kin()
  assert approx_equal(e_kin_1(), 60.9875715394)
  tt.fix_near_singular_hinges(sites=sites)
  assert tt.cluster_manager.fixed_hinges == [(2,3)]
  assert tt.cluster_manager.clusters == [[0,1,2,3,4]]
  assert approx_equal(e_kin_1(), 1.00004830172, eps=1e-10)
  #
  sites.append(matrix.col((3,0,0)))
  labels.append("5")
  masses.append(1)
  edge_list.append((4,5))
  tt = tardy_tree.construct(n_vertices=len(sites), edge_list=edge_list)
  tt.build_tree()
  assert tt.cluster_manager.clusters == [[0,1,2,3], [4], [5]]
  assert approx_equal(e_kin_1(), 9.55508653428)
  tt.fix_near_singular_hinges(sites=sites)
  assert tt.cluster_manager.fixed_hinges == [(2,3), (3,4)]
  assert tt.cluster_manager.clusters == [[0,1,2,3,4,5]]
  assert approx_equal(e_kin_1(), 1.00005333167, eps=1e-10)
  #
  sites.append(matrix.col((3-x,-y,0)))
  assert approx_equal(abs(sites[5] - sites[6]), 1)
  labels.append("6")
  masses.append(1)
  edge_list.append((5,6))
  tt = tardy_tree.construct(n_vertices=len(sites), edge_list=edge_list)
  tt.build_tree()
  assert tt.cluster_manager.clusters == [[0,1,2,3], [4], [5], [6]]
  assert approx_equal(e_kin_1(), 0.99994375467)
  tt.fix_near_singular_hinges(sites=sites)
  assert tt.cluster_manager.fixed_hinges == [(2,3), (3,4)]
  assert tt.cluster_manager.clusters == [[0,1,2,3,4,5], [6]]
  assert approx_equal(e_kin_1(), 1.0000438095, eps=1e-10)

def exercise_linear_velocity_manipulations(tardy_model):
  for nosiet in [None, tardy_model.number_of_sites_in_each_tree()]:
    tardy_model.assign_random_velocities(e_kin_target=17)
    if (tardy_model.degrees_of_freedom == 0):
      assert approx_equal(tardy_model.e_kin(), 0)
      assert tardy_model.mean_linear_velocity(
        number_of_sites_in_each_tree=None) is None
    else:
      assert approx_equal(tardy_model.e_kin(), 17)
      mlv = tardy_model.mean_linear_velocity(
        number_of_sites_in_each_tree=nosiet)
      if (mlv is not None):
        assert is_above_limit(value=abs(mlv), limit=1e-3)
        tardy_model.subtract_from_linear_velocities(
          number_of_sites_in_each_tree=nosiet, value=mlv)
        mlv = tardy_model.mean_linear_velocity(
          number_of_sites_in_each_tree=nosiet)
        assert approx_equal(mlv, (0,0,0))

def exercise_fixed_vertices_special_cases():
  tardy_models = []
  """
          2
         /
    0---1
  """
  x = 0.5*3**0.5
  y = 0.5
  sites = matrix.col_list([
    (0,0,0),
    (1,0,0),
    (1+x,y,0)])
  edge_list = [(0,1),(1,2)]
  for i,j in edge_list:
    assert approx_equal(abs(sites[i]-sites[j]), 1)
  labels = [str(i) for i in range(len(sites))]
  masses = [1] * len(sites)
  #
  tt = tardy_tree.construct(
    sites=sites,
    edge_list=edge_list,
    fixed_vertex_lists=[])
  assert tt.cluster_manager.clusters == [[0,1,2]]
  tm = construct_tardy_model(
    labels=labels, sites=sites, masses=masses, tardy_tree=tt)
  tardy_models.append(tm)
  assert len(tm.bodies) == 1
  assert tm.bodies[0].joint.degrees_of_freedom == 6
  exercise_linear_velocity_manipulations(tardy_model=tm)
  #
  expected_e_kin_1 = [
    1.00009768395,
    1.00002522865,
    1.00000107257,
    1.0,
    1.0,
    1.0,
    0.0]
  rnd = random.Random(0)
  for i_fv,fixed_vertices in enumerate([[0], [1], [2],
                                        [0,1], [0,2], [1,2],
                                        [0,1,2]]):
    tt = tardy_tree.construct(
      sites=sites,
      edge_list=edge_list,
      fixed_vertex_lists=[fixed_vertices])
    assert tt.cluster_manager.clusters == [[0,1,2]]
    tm = construct_tardy_model(
      labels=labels, sites=sites, masses=masses, tardy_tree=tt)
    tardy_models.append(tm)
    assert len(tm.bodies) == 1
    assert tm.bodies[0].joint.degrees_of_freedom \
        == [3,1,0][len(fixed_vertices)-1]
    tm.assign_random_velocities(e_kin_target=1, random_gauss=rnd.gauss)
    if (len(fixed_vertices) != 3):
      assert approx_equal(tm.e_kin(), 1, eps=1e-10)
    else:
      assert approx_equal(tm.e_kin(), 0, eps=1e-10)
    tm.dynamics_step(delta_t=0.01)
    assert approx_equal(tm.e_kin(), expected_e_kin_1[i_fv], eps=1e-10)
    exercise_linear_velocity_manipulations(tardy_model=tm)
  #
  sites[2] = matrix.col([2,0,0])
  assert approx_equal((sites[0]-sites[1]).cos_angle(sites[2]-sites[1]), -1)
  for fixed_vertices in [[0,1], [0,2], [1,2]]:
    tt = tardy_tree.construct(
      sites=sites,
      edge_list=edge_list,
      fixed_vertex_lists=[fixed_vertices])
    assert tt.cluster_manager.clusters == [[0,1,2]]
    tm = construct_tardy_model(
      labels=labels, sites=sites, masses=masses, tardy_tree=tt)
    tardy_models.append(tm)
    assert len(tm.bodies) == 1
    assert tm.bodies[0].joint.degrees_of_freedom == 0
    exercise_linear_velocity_manipulations(tardy_model=tm)
  return tardy_models

def exercise_tardy_model(out, n_dynamics_steps, delta_t, tardy_model):
  tardy_model.check_d_e_pot_d_q()
  e_pots = flex.double([tardy_model.e_pot()])
  e_kins = flex.double([tardy_model.e_kin()])
  for i_step in range(n_dynamics_steps):
    tardy_model.dynamics_step(delta_t=delta_t)
    e_pots.append(tardy_model.e_pot())
    e_kins.append(tardy_model.e_kin())
  e_tots = e_pots + e_kins
  tardy_model.check_d_e_pot_d_q()
  print("degrees of freedom:", tardy_model.degrees_of_freedom, file=out)
  print("energy samples:", e_tots.size(), file=out)
  print("e_pot min, max:", min(e_pots), max(e_pots), file=out)
  print("e_kin min, max:", min(e_kins), max(e_kins), file=out)
  print("e_tot min, max:", min(e_tots), max(e_tots), file=out)
  print("start e_tot:", e_tots[0], file=out)
  print("final e_tot:", e_tots[-1], file=out)
  ave = flex.sum(e_tots) / e_tots.size()
  range_ = flex.max(e_tots) - flex.min(e_tots)
  if (ave == 0): relative_range = 0
  else:          relative_range = range_ / ave
  print("ave:", ave, file=out)
  print("range:", range_, file=out)
  print("relative range:", relative_range, file=out)
  print(file=out)
  out.flush()
  return relative_range

def exercise_dynamics_quick(
      out, tardy_model, n_dynamics_steps, delta_t=0.0001):
  relative_range = exercise_tardy_model(
    out=out,
    n_dynamics_steps=n_dynamics_steps,
    delta_t=delta_t,
    tardy_model=tardy_model)
  if (out is not sys.stdout):
    if (len(tardy_model.tardy_tree.cluster_manager.loop_edges) == 0):
      assert relative_range < 1.e-5
    else:
      assert relative_range < 2.e-4
  print(file=out)

def exercise_minimization_quick(out, tardy_model, max_iterations=5):
  print("Minimization:", file=out)
  print("  start e_pot:", tardy_model.e_pot(), file=out)
  e_pot_start = tardy_model.e_pot()
  tardy_model.minimization(max_iterations=max_iterations)
  print("  final e_pot:", tardy_model.e_pot(), file=out)
  e_pot_final = tardy_model.e_pot()
  if (out is not sys.stdout):
    assert e_pot_final < e_pot_start * 0.7
  print(file=out)

def construct_tardy_model(
      labels,
      sites,
      masses,
      tardy_tree):
  cm = tardy_tree.cluster_manager
  return tardy_model(
    labels=labels,
    sites=sites,
    masses=masses,
    tardy_tree=tardy_tree,
    potential_obj=potential_object(
      sites=sites,
      wells=sites,
      restraint_edges=cm.loop_edges+cm.loop_edge_bendings))

def exercise_with_tardy_model(out, tardy_model, n_dynamics_steps):
  tardy_model.tardy_tree.show_summary(out=out, vertex_labels=None)
  #
  ri = tardy_model.root_indices()
  assert len(ri) == 1
  nosiet = tardy_model.number_of_sites_in_each_tree()
  assert len(nosiet) == 1
  assert nosiet[0][0] == ri[0]
  assert nosiet[0][1] == len(tardy_model.sites)
  somiet = tardy_model.number_of_sites_in_each_tree()
  assert len(somiet) == 1
  assert somiet[0][0] == ri[0]
  assert somiet[0][1] == len(tardy_model.sites)
  #
  exercise_qd_e_kin_scales(tardy_model=tardy_model)
  exercise_random_velocities(tardy_model=tardy_model)
  exercise_linear_velocity_manipulations(tardy_model=tardy_model)
  tardy_model.assign_random_velocities(e_kin_target=1)
  assert approx_equal(tardy_model.e_kin(), 1)
  exercise_dynamics_quick(
    out=out, tardy_model=tardy_model, n_dynamics_steps=n_dynamics_steps)
  exercise_minimization_quick(out=out, tardy_model=tardy_model)

n_test_models = len(test_cases_tardy_pdb.test_cases)

def get_test_model_by_index(i, fixed_vertex_lists=[]):
  tc = test_cases_tardy_pdb.test_cases[i]
  tt = tc.tardy_tree_construct(fixed_vertex_lists=fixed_vertex_lists)
  return construct_tardy_model(
    labels=tc.labels,
    sites=tc.sites,
    masses=[1.0]*len(tc.sites),
    tardy_tree=tt)

test_case_5_fixed_vertices_expected_dof = [
  ([0,16,17], 2),
  ([16,17], 5),
  ([12], 8)]

def run(args):
  assert len(args) in [0,1]
  if (len(args) == 0):
    n_dynamics_steps = 100
    out = null_out()
  else:
    n_dynamics_steps = max(1, int(args[0]))
    out = sys.stdout
  show_times_at_exit()
  #
  exercise_accumulate_in_each_tree()
  exercise_near_singular_hinges()
  exercise_fixed_vertices_special_cases()
  #
  if (1):
    for i in range(n_test_models):
      print("test model index:", i, file=out)
      tardy_model = get_test_model_by_index(i=i)
      exercise_with_tardy_model(
        out=out, tardy_model=tardy_model, n_dynamics_steps=n_dynamics_steps)
      if (i == 0):
        assert tardy_model.degrees_of_freedom == 3
        fixed_vertices = [0]
        print("test model index:", i, \
          "fixed_vertices:", fixed_vertices, file=out)
        tardy_model = get_test_model_by_index(
          i=i, fixed_vertex_lists=[fixed_vertices])
        assert tardy_model.degrees_of_freedom == 0
      elif (i == 5):
        assert tardy_model.degrees_of_freedom == 11
        for fixed_vertices,expected_dof in \
              test_case_5_fixed_vertices_expected_dof:
          print("test model index:", i, \
            "fixed_vertices:", fixed_vertices, file=out)
          tardy_model = get_test_model_by_index(
            i=i, fixed_vertex_lists=[fixed_vertices])
          assert tardy_model.degrees_of_freedom == expected_dof
          exercise_with_tardy_model(
            out=out,
            tardy_model=tardy_model,
            n_dynamics_steps=n_dynamics_steps)
  #
  print("OK")

if (__name__ == "__main__"):
  run(sys.argv[1:])


import logging
from pytube import YouTube

def ytpytubeStreamObject(yt):
    ytStream=yt.streams
    cnt=0
    for stream in ytStream:
        print(stream)
        if not stream.is_adaptive :

            cnt+=1
    print(cnt)
if __name__ == '__main__':
    path = "C:\\users\\cg398\\Desktop"
    yt = YouTube("https://www.youtube.com/watch?v=IKfkSHpkBpw&feature=youtu.be")


from PIL import Image, ImageDraw
import glob
import random
import os

classes = ['0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'a', 'b', 'c', 'd', 'e',
               'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r', 's', 't',
               'u', 'v', 'w', 'x', 'y', 'z', 'dxb', 'uaq', 'shj', 'auh', 'ajm', 'fuj', 'rak',
               'prefix', 'platenum', 'others', 'taxi', 'dubai_police', 'consulate']

print(len(classes))
def yolo_to_xml_bbox(bbox, w, h):
    # x_center, y_center width heigth
    w_half_len = (bbox[2] * w) / 2
    h_half_len = (bbox[3] * h) / 2
    xmin = int((bbox[0] * w) - w_half_len)
    ymin = int((bbox[1] * h) - h_half_len)
    xmax = int((bbox[0] * w) + w_half_len)
    ymax = int((bbox[1] * h) + h_half_len)
    return [xmin, ymin, xmax, ymax]

def draw_image(img, bboxes, classes_idxes):
    draw = ImageDraw.Draw(img)
    for bbox, class_idx in zip(bboxes,classes_idxes):
        draw.rectangle(bbox, outline="red", width=2)
        print(class_idx)
        if not class_idx == '100':
            class_name = classes[int(class_idx)]
            draw.text((bbox[0], bbox[1] - 10), class_name.upper(), fill=(0, 255, 0))
    img.save("example.jpg")
    img.show()

# image_filename = "Data/OCR_Data/Cropped_Plates_Img/0010100015220906171354.jpg"
# label_filename = "Data/OCR_Data/Cropped_Plates_Anno/0010100015220906171354.txt"

# img_dir = 'Data/OCR_Data/test/images/'
# anno_dir = 'Data/OCR_Data/test/labels/'
img_dir = 'Data/vizthis/img'
anno_dir = 'Data/vizthis/labels'
file_list = glob.glob(img_dir + "/*")

image_filename = random.choice(file_list)
label_filename = os.path.join(anno_dir,
                              os.path.basename(image_filename).split('.')[0] + '.txt')

bboxes = []
classes_idxs = []
img = Image.open(image_filename)

#0010200034220906192858
# img = Image.open("Data/test/images/0010200034220906192858.jpg")
# label_filename = "Data/test/labels/0010200034220906192858.txt"

print(label_filename)
with open(label_filename, 'r', encoding='utf8') as f:
    for line in f:

        data = line.strip().split(' ')
        class_idx = data[0]
        print(data, classes[int(class_idx)])
        # bbox = [float(x) for x in data[1:] if data[0] == '5']
        bbox = [float(x) for x in data[1:]]

        # if class_idx != '2' and class_idx != '0' and class_idx != '43' and class_idx != '44' and class_idx != '7':

        if class_idx == '36':
            classes_idxs.append(class_idx)
            if bbox:
                bboxes.append(yolo_to_xml_bbox(bbox, img.width, img.height))
draw_image(img, bboxes, classes_idxs)


import solution
import unittest
from ddt import ddt, data, unpack

@ddt
class CakeTestCase(unittest.TestCase):

    @unpack
    @data(
        ("abc", 1),
        ("abcdefabc", 1),
        ("abccbaabccba", 2),
        ("abcabcabcabc", 4)
    )
    def test_answer(self, input_string, expected):
        self.assertEquals(expected, solution.answer(input_string))

if __name__ == '__main__':
    unittest.main()


from django.test import TestCase

from server.forms.random_item_form import RandomItemDrawForm

class RandomItemDrawFormTest(TestCase):
    def valid_ok_test(self):
        """RandomItemDrawForm: Correct draw is valid"""
        form_data = {'number_of_results': 1, 'items': ['a', 'b', 'c']}
        form = RandomItemDrawForm(data=form_data)
        self.assertTrue(form.is_valid())

    def not_enough_items_ko_test(self):
        """RandomItemDrawForm: Not enough items is not valid"""
        form_data = {'number_of_results': 1, 'items': []}
        form = RandomItemDrawForm(data=form_data)
        self.assertFalse(form.is_valid())

    def not_enough_results_ko_test(self):
        """RandomItemDrawForm: Number of items > 0 """
        form_data = {'number_of_results': 0}
        form = RandomItemDrawForm(data=form_data)
        self.assertFalse(form.is_valid())


import boto3
import pickle

from flask import Flask, request, json
from sklearn.ensemble import GradientBoostingClassifier
from sklearn.externals import joblib

MODEL_FILE_NAME = 'model.pkl'

app = Flask(__name__)

@app.route('/', methods=['POST'])
def index():
    # Parse request body for model input
    body_dict = request.get_json(silent=True)
    data = body_dict['data']

    # Load model
    model = joblib.load(MODEL_FILE_NAME)
    # Make prediction
    prediction = model.predict(data).tolist()
    # Respond with prediction result
    result = {'prediction': prediction}
    return json.dumps(result)

if __name__ == '__main__':
    # listen on all IPs
    app.run(host='0.0.0.0')


import logging, requests
from fake_useragent import UserAgent

class Http(object):

    log = logging.getLogger(__name__)

    def session(self, proxies={}, cookies=None):

        headers = {'User-Agent': UserAgent(verify_ssl=False).chrome}

        session = requests.session()

        session.headers.update(headers)

        session.proxies=proxies

        if cookies: session.cookies=cookies

        return session

    def client(self, session, url, method='GET', data=None):

        try:

            result = session.request(method, url, data=data)

            code = result.status_code

            if code in [200, 201]: return result.text

            self.log.warning('HTTP状态:[{}], {}'.format(code, url))

        except Exception as e: self.log.warning('错误信息:',e)

        return ''


from selenium import webdriver
import unittest

class OpenTest(unittest.TestCase):
    def setUp(self):
        self.browser = webdriver.Safari()

    def testOpen(self):
        self.browser.get('http://127.0.0.1:8000/fitness')
        self.assertIn('Ты лучше', self.browser.title)

if __name__ == '__main__':
    unittest.main()


import pandas as pd
import matplotlib.pyplot as plt
import numpy as np

#load the data
data = pd.read_csv('blocktimes_raw.csv')
data['timestamp'] = pd.to_datetime(data['timestamp'])
data['blocktime'] = data['timestamp'].diff()

#convert blocktime to seconds
data['blocktime'] = data['blocktime'].dt.total_seconds()

#drop data for blocks before 15050001
data = data.loc[data['blocknumber'] >= 15050001]

#group the data by every 1000 blocks and calculate the mean of dif
data_1000 = data.groupby(data['blocknumber']//1000).blocktime.mean()
#calculate the standard deviation of dif every 1000 blocks
data_1000_sd = data.groupby(data['blocknumber']//1000).blocktime.std()

#create a scatter plot of blocktime on the y-axis and number on the x-axis

plt.scatter(data_1000.index/1000, data_1000.values, s = 0.75, c='black')
plt.axvline(x = 15.537393, color = 'b')
plt.text(15.4, 15.7, 'Ethereum Merge', verticalalignment = 'center_baseline', c='b')
plt.xlabel('Block Number (in millions)')
plt.ylabel('Average Block Time (1000 block increments)')
#add a regression line if blockgroup is less than 15.537 and greater than or equal to 15.537

x = data_1000.index/1000
y = data_1000.values

mask = x < 15.537393
x1 = x[mask]
y1 = y[mask]
m1, b1 = np.polyfit(x1, y1, 1)
print(m1, b1)
x2 = x[~mask]
y2 = y[~mask]
m2, b2 = np.polyfit(x2, y2, 1)
print(m2, b2)
plt.plot(x1, m1*x1 + b1, color = 'r', label = 'blockgroup < 15.537')
plt.plot(x2, m2*x2 + b2, color = 'g', label = 'blockgroup >= 15.537')

#save figure
plt.savefig('blocktimeavg.png', dpi = 300)
plt.show()

#create a scatter plot of moving standard deviation of blocktime on the y-axis and number on the x-axis
#do not include if the standard deviation is NaN
data_1000_sd = data_1000_sd.dropna()

plt.scatter(data_1000_sd.index/1000, data_1000_sd.values, s = 0.75, c='black')
plt.axvline(x = 15.537, color = 'b')
plt.text(15.4, 16.7, 'Ethereum Merge', verticalalignment = 'center_baseline', c='b')
plt.xlabel('Block Number (in millions)')
plt.ylabel('Standard Deviation of Block Time (1000 block increments)')
#add a regression line if blockgroup is less than 15.537 and greater than or equal to 15.537
import numpy as np
import matplotlib.pyplot as plt

x = data_1000_sd.index/1000
y = data_1000_sd.values

mask = x < 15.537
x1 = x[mask]
y1 = y[mask]
m1, b1 = np.polyfit(x1, y1, 1)
print(m1, b1)
x2 = x[~mask]
y2 = y[~mask]
m2, b2 = np.polyfit(x2, y2, 1)
print(m2, b2)
plt.plot(x1, m1*x1 + b1, color = 'r', label = 'blockgroup < 15.537')
plt.plot(x2, m2*x2 + b2, color = 'g', label = 'blockgroup >= 15.537')

# save figure so it can be used in latex
plt.savefig('blocktimestd.png', dpi=300)
plt.show()


import random
import os
import json
import networkx as nx
import matplotlib.pyplot as plt

MAX_POPULATION = 8
INITIAL_POPULATION_PER_REGION = 4

class Region:

    country_id = -1
    id = -1
    adjacency_vector = []
    population = 0

    def __init__(self):
        pass

class Arena:

    regions = []
    regions_count = 0
    adjacency_matrix = []
    countries = 0
    country_regions = []
    country_scores = []
    selected_region = -1
    scores = []
    G = 0
    _op = 0

    active_player = 0
    max_players = 0
    round = 0

    def __init__(self):
        pass

    def _throw_dices(self, dices):
        result = 0
        for dice in range(dices):
            result = result + random.randint(1, 6)
        return dices  # debug

    def act(self, region_id, end_turn=False):
        if end_turn:
            self.end_turn()
            return 0

        if self.selected_region == -1:
            if self.regions[region_id].country_id != self.active_player or region_id == self.selected_region:
                return -1
            else:
                self.selected_region = region_id
                # print('SEL: ', region_id, 'CO:', self.active_player)
                return 0

        src_region_id = self.selected_region
        target_region_id = region_id

        src_region = self.regions[src_region_id]
        target_region = self.regions[target_region_id]

        if target_region.country_id == self.active_player:
            return -1

        if src_region.adjacency_vector[target_region_id] == 0:
            return -1

        # print('BAT: ', self.selected_region, '->', target_region_id)

        score_before = self.scores[src_region.country_id]

        src = self._throw_dices(src_region.population)
        target = self._throw_dices(target_region.population)

        if src > target:
            target_region.population = src_region.population - 1
            src_region.population = 1

            self.country_regions[target_region.country_id].remove(
                target_region_id)
            target_region.country_id = src_region.country_id
            self.country_regions[target_region.country_id].extend([
                target_region_id])
            self._calculate_scores()
        else:
            src_region.population = 1

        score_after = self.scores[src_region.country_id]
        score_diff = score_after - score_before
        self.selected_region = -1
        return score_after * 0.2
        # if score_diff > 0:
        #     return 0.2 * score_diff
        # else:
        #     return 0

    def new_game(self):
        self._build_country_map()
        self._plant_population()
        self._calculate_scores()
        self.max_players = len(self.scores)
        # self.print_country(0)

    def end_turn(self):
        tod = self.scores[self.active_player]
        rgs = self.country_regions[self.active_player]

        if len(rgs) > 0:
            gltod = 1000
            while tod > 0:
                idx = 0
                gltod -= 1
                if gltod <= 0:
                    tod = 0
                if len(rgs) > 1:
                    idx = rgs[random.randint(
                        0, len(rgs) - 1)]
                canbe = False
                for i in range(len(rgs)):
                    if self.regions[rgs[i]].population < MAX_POPULATION:
                        canbe = True
                if not canbe:
                    tod = 0
                if self.regions[idx].population < MAX_POPULATION:
                    tod = tod - 1
                    self.regions[idx].population = self.regions[idx].population + 1

        self.active_player = self.active_player + 1
        if self.active_player >= self.max_players:
            self.active_player = 0
        self.selected_region = -1
        pass

    def draw_country_network(self, country_id, file_name, label=''):
        nodelist = []
        edgelist = []
        for i in range(self.regions_count):
            nodelist.extend([i])
        for region_id in self.country_regions[country_id]:
            # nodelist.extend([region_id])
            for edge in self.G.edges(region_id):
                if self.country_regions[country_id].count(edge[0]) > 0 and self.country_regions[country_id].count(edge[1]) > 0:
                    edgelist.extend([edge])

        options = {
            'node_size': 300,
            'width': 1,
            'node_color': '#EFEFEF',
            'font_size': 10,
            'nodelist': list(nodelist),
            'edgelist': list(edgelist),
            'with_labels': True
        }
        plt.get_current_fig_manager().canvas.figure.clear()
        nx.draw_kamada_kawai(self.G, **options)  # best so far
        plt.suptitle(label, y=0.05, x=0.3)
        plt.savefig(file_name)

    def draw_network(self):
        options = {
            'node_size': 300,
            'width': 1,
            'node_color': '#EFEFEF',
            'font_size': 10,
            'with_labels': True
        }
        nx.draw_kamada_kawai(self.G, **options)  # best so far
        plt.get_current_fig_manager().canvas.figure.clear()
        plt.savefig('env/graph.png')

    def _get_connections(self, node, nodes):
        cid = 0
        connections = 0
        for i in self.adjacency_matrix[node]:
            self._op += 1
            if nodes.count(cid) == 0 and i == 1 and self.regions[cid].country_id == self.regions[node].country_id:
                nodes.extend([cid])
                connections = connections + 1 + \
                    self._get_connections(cid, nodes)
            cid += 1
        return connections

    def _calculate_scores(self, country_id=-1):
        if country_id == -1:
            self.scores = list(map(lambda x: 1, range(self.countries)))
        for region in self.regions:
            if country_id >= 0 and region.country_id != country_id:
                continue
            score = self._get_connections(region.id, [])
            if score > self.scores[region.country_id]:
                self.scores[region.country_id] = score

    def _plant_population(self):
        for country_id in range(self.countries):
            pool = len(self.country_regions[country_id]
                       ) * INITIAL_POPULATION_PER_REGION
            while pool > 0:
                for region_id in self.country_regions[country_id]:
                    region = self.regions[region_id]
                    if region.population == 0:
                        region.population = 1
                        pool -= 1
                        continue
                    if pool > 0 and region.population < MAX_POPULATION and random.random() > 0.5:
                        pool -= 1
                        region.population += 1

    def _build_country_map(self):
        for region in self.regions:
            self.country_regions[region.country_id].extend([region.id])

    def print_country(self, country_id):
        pops = []
        for i in range(len(self.regions)):
            if self.regions[i].country_id == self.active_player:
                pops.append(self.regions[i].population)
        print('CO: ', country_id, ' Score: ',
              self.scores[country_id], ' Pops: ', pops, ' / ', sum(pops))

    def get_country_sizes(self):
        rets = []
        for i in range(self.countries):
            rets.append(len(self.country_regions[i]))
        return rets

    def load_from_file(self, file_name):
        self.country_regions = []
        self.country_scores = []
        self.regions = []
        self.adjacency_matrix = []
        self.active_player = 0
        self.selected_region = -1

        with open(file_name) as f:
            data = json.load(f)
        self.adjacency_matrix = data['full']

        self.G = nx.Graph()
        self.G.add_nodes_from(range(0, len(self.adjacency_matrix)))

        region_id = 0
        max_country = 0
        for adjacency_vector in self.adjacency_matrix:
            region = Region()
            region.id = region_id
            region.country_id = data['country_map'][str(region_id)] - 1
            region.adjacency_vector = adjacency_vector

            cid = 0
            for connected in adjacency_vector:
                if connected:
                    self.G.add_edge(region_id, cid)
                cid += 1

            self.regions.extend([region])
            if region.country_id > max_country:
                max_country = region.country_id
            region_id += 1

        self.countries = max_country + 1
        for country_id in range(0, self.countries):
            self.country_regions.extend([[]])
            self.country_scores.extend([0])

        self.regions_count = len(self.regions)

if __name__ == '__main__':
    arena = Arena()
    print('SelfTest', os.getcwd())
    arena.load_from_file(os.getcwd().replace('/env', '') + '/models/map.json')
    print('Regions: ', len(arena.regions), ' / Countrzises: ', arena.countries)
    arena.new_game()
    print('Reward: ', arena.act(29, 30))


'''
Created on Apr 10, 2012

@organization: cert.org
'''
import unittest
from certfuzz.fuzztools import text
import math
import tempfile
import shutil
import os
import random

class Test(unittest.TestCase):

    def setUp(self):
        self.tempdir = tempfile.mkdtemp()

    def tearDown(self):
        shutil.rmtree(self.tempdir)

    def test__pattern(self):
        iterables = ['ABC', 'abc', '012']
        for length in range(3, 1000):
            pat = text._pattern(iterables, length)
            self.assertEqual(length, len(pat))
            self.assertTrue(pat.startswith('Aa0'))
            if length > 100:
                self.assertTrue('Cc2' in pat)

        # make sure it rolls over
        # each iterable has 3 possible values
        # and there are 3 iterables thus 3 chars per token
        rollover_pos = 3 * 3 * 3 * 3
        # so we want to overshoot by one token
        # to ensure that we rollover to the start token Aa0 again
        rollover_str = text._pattern(iterables, rollover_pos + 3)
        self.assertTrue(rollover_str.endswith('Cc2Aa0'))

    def test_metasploit_pattern_orig(self):
        for length in range(3, 1000):
            pat = text.metasploit_pattern_orig(length)
            self.assertEqual(length, len(pat))
            self.assertTrue(pat.startswith('Aa0'))
            if length > 101:
                self.assertEqual('Ad3', pat[99:102])

    def test_metasploit_pattern_extended(self):
        for length in range(6, 1000):
            pat = text.metasploit_pattern_extended(length)
            self.assertEqual(length, len(pat))
            self.assertTrue(pat.startswith('AAaa00'))
            if length > 101:
                self.assertEqual('AAaa16', pat[96:102])

    def test__enumerate_string(self):
        s = 'x____' * 1000

        occurrences = sorted(random.sample(population=range(0, len(s), 5), k=102))
        result = text._enumerate_string(s, occurrences)
        # length should be unchanged
        self.assertEqual(len(s), len(result))

        # convert result to string
        result = str(result)
        self.assertNotEqual(s, result)
        # check 1 digit
        self.assertTrue('____1____' in result)
        # check 2 digit
        self.assertTrue('___10___' in result)
        # check 3 digit
        self.assertTrue('__100__' in result)

    def test_enumerate_string(self):
        (fd, f) = tempfile.mkstemp(suffix='.foo', dir=self.tempdir)
        os.write(fd, 'AAAAxxxAAAAxxxxxAAAAxAAAA')
        os.close(fd)
        root, ext = os.path.splitext(f)
        expected_newpath = '{}-enum{}'.format(root, ext)

        newpath = text.enumerate_string(f, 'AAAA')
        self.assertTrue(newpath.endswith('-enum.foo'))

        self.assertEqual(expected_newpath, newpath)
        self.assertTrue(os.path.exists(expected_newpath))

        with open(newpath, 'rb') as fp:
            content = fp.read()
            self.assertEqual('0AAAxxx1AAAxxxxx2AAAx3AAA', content)
        os.remove(newpath)
        os.remove(f)

if __name__ == "__main__":
    # import sys;sys.argv = ['', 'Test.testName']
    unittest.main()


from django.urls import path
from product.views import productdetail, ProductListView, ProductDetailView,export,add_wishlist,my_wishlist,deleteFromWishlist,item

urlpatterns = [
    # path('productdetail/',productdetail,name="productdetail"),
    path('productlist/',ProductListView.as_view(),name="productlist"),
    path('productdetail/<int:pk>/',ProductDetailView.as_view(),name='productdetail'),
    path('export/', export, name='export'),
    path('add-wishlist',add_wishlist,name='add_wishlist'),
    path('my-wishlist',my_wishlist,name='my_wishlist'),
    path('delete-from-wishlist/',deleteFromWishlist,name="delete-from-wishlist"),
    path('delete/',item,name="delete"),

    # path('my-wishlist',my_wishlist,name='my_wishlist'),

]


import requests
import time
import json
import dateutil.parser as dateutilparser
from bs4 import BeautifulSoup
from dotenv import load_dotenv
from notify import notify

load_dotenv()

forums = ["https://forums.redflagdeals.com/hot-deals-f9/?st=0&rfd_sk=tt&sd=d"]
FILE_NAME = 'cache-rfd.json'

def form_full_rfd_url(relative_path):
    return 'https://forums.redflagdeals.com' + relative_path

try:
    with open(FILE_NAME, 'r') as f:
        cache = json.load(f)
except:
    cache = {}

try:
    for forum in forums:
        html_text = requests.get(forum).text
        soup = BeautifulSoup(html_text, 'html.parser')
        soup = soup.select('ul.topiclist.topics.with_categories')[0]

        thread_tags = soup.select('li.row.topic')

        for thread_tag in thread_tags:
            # Ignore sticky threads
            if thread_tag.find(class_='sticky'):
                continue

            id = thread_tag['data-thread-id']
            if id in cache:
                continue

            try:
                # Extract publish time
                post_time = dateutilparser.parse(thread_tag.select('span.first-post-time')[0].text).timestamp()
                time_str = time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(post_time))

                # Thread link
                title_link_tag = thread_tag.select('a.topic_title_link')[0]
                link = form_full_rfd_url(title_link_tag['href'])

                title = thread_tag.select('h3.topictitle')[0].text.strip().replace('\n', '')

                message = f"{time_str}: {title}"
                message += "\n\n"
                message += link
                message += "\n\n"

                notify(message)
                cache[id] = post_time
            except:
                pass
finally:
    with open(FILE_NAME, 'w') as outfile:
        json.dump(cache, outfile)


import numpy as np
import cv2 as cv
from matplotlib import pyplot as plt

img1 = cv.imread('../../images/free-photo-Statue-38-m.jpg', 1)          # the same size as photo below
img2 = cv.imread('../../images/free-photo-twin-fowers-407-m.jpg', 1)

dst = cv.addWeighted(img1, 0.7, img2, 0.5, 0)           # marge two photo with any weight

# cv.imshow('dst',dst)
cv.waitKey(0)
cv.destroyAllWindows()

#***************************************Add two photo with foreground and background************************
print('******************')
img1 = cv.imread('../../images/free-photo-Statue-38-m.jpg', 1)  # small
img2 = cv.imread('../../images/ludzie_krajobraz.jpg', 1)        # big

rows, cols, channels = img1.shape   # get photo params
roi = img2[0:rows, 0:cols]          # make Region Of Interest (ROI) at secont pic

# Now create a mask of logo and create its inverse mask also
img1gray = cv.cvtColor(img1, cv.COLOR_BGR2GRAY)
ret, mask = cv.threshold(img1gray, 120, 255, cv. THRESH_BINARY)     # make color photo to binary photo
mask_inv = cv.bitwise_not(mask)

# black out the area of small pic in ROI
img2_bg = cv.bitwise_and(roi, roi, mask=mask_inv)       # place at big photo binary mask from small photo

# take only region of logo from image
img1_fg = cv.bitwise_and(img1, img1, mask=mask)         # place at small photo binary mask from small photo

# put logo in ROI and modify the main image
# add background and foreground. Black color has value of '0', so add black and blue = blue :)
dst = cv.add(img2_bg, img1_fg)

cv.imshow('img2_bg', img2_bg)
cv.imshow('img1_fg', img1_fg)
cv.imshow('dst', dst)
cv.waitKey(0)

img2[0:rows, 0:cols] = dst

cv.imshow('res', img2)
cv.waitKey(0)
cv.destroyAllWindows()


import re
from enum import Enum
from typing import Counter, Dict, Iterator, List

class Emotions(Enum):
    ANGER = 1 << 0
    ANTICIPATION = 1 << 1
    DISGUST = 1 << 2
    FEAR = 1 << 3
    JOY = 1 << 4
    NEGATIVE = 1 << 5
    POSITIVE = 1 << 6
    SADNESS = 1 << 7
    SURPRISE = 1 << 8
    TRUST = 1 << 9
    ANY = 1 << 10
    def __int__(self):
        return self.value

dic: Dict[str, List[Emotions]] = {}
dicBitmap: Dict[str, int] = {}

def getEmotionName(emotion: Emotions) -> str:
    if emotion == Emotions.ANGER:
        return "anger"
    if emotion == Emotions.ANTICIPATION:
        return "anticipation"
    if emotion == Emotions.DISGUST:
        return "disgust"
    if emotion == Emotions.FEAR:
        return "fear"
    if emotion == Emotions.JOY:
        return "joy"
    if emotion == Emotions.NEGATIVE:
        return "negative"
    if emotion == Emotions.POSITIVE:
        return "positive"
    if emotion == Emotions.SADNESS:
        return "sadness"
    if emotion == Emotions.SURPRISE:
        return "surprise"
    if emotion == Emotions.TRUST:
        return "trust"
    if emotion == Emotions.ANY:
        return "analized"
    return "unknown"

def initEmotionLexicon(lang = 'en'):
    # # English
    # emotionOrder = [
    #     Emotions.ANGER, Emotions.ANTICIPATION, Emotions.DISGUST, Emotions.FEAR, Emotions.JOY,
    #     Emotions.NEGATIVE, Emotions.POSITIVE,Emotions.SADNESS, Emotions.SURPRISE, Emotions.TRUST
    # ]
    # path = './assets/NRC-Emotion-Lexicon-Wordlevel-v0.92.txt'
    # with open(path) as file:
    #     lines = [l for l in file]
    #     for i in range(0, len(lines), 10):
    #         wordLine = [l for l in lines[i:i + 10]]
    #         term = wordLine[0].split('\t')[0]
    #         emotions = [Emotions.ANY]
    #         for j in range(10):
    #             if int(wordLine[j].split('\t')[2][0]) == 1:
    #                 emotions.append(emotionOrder[j])
    #         dic[term] = emotions
    #         # bitmap = 1 << 10
    #         # for j in range(10):
    #         #     bitmap += int(wordLine[j].split('\t')[2][0]) << j
    #         # dic[term] = bitmap

    # Other languages
    emotionOrder = [
        Emotions.POSITIVE, Emotions.NEGATIVE, Emotions.ANGER, Emotions.ANTICIPATION, Emotions.DISGUST,
        Emotions.FEAR, Emotions.JOY, Emotions.SADNESS, Emotions.SURPRISE, Emotions.TRUST
    ]
    path = './assets/NRC-Emotion-Lexicon-v0.92-In105Languages-Nov2017Translations.csv'
    with open(path) as file:
        lines = [l for l in file]
        # find header column
        header = lines[0].strip('\n').split(',')[:-10]
        langCol = -1
        for i, langHeader in enumerate(header):
            if langHeader.split('(')[1].split(')')[0] == lang:
                print(langHeader)
                langCol = i
                break
        try:
            if langCol < 0:
                raise Exception('Lang not found')
        except Exception as ex:
            print(ex)
            exit(1)

        for l in lines[1:]:
            d = l.strip('\n').split(',')
            term = d[langCol]
            if lang == 'it' and (term == 'mar' or term == 'ago'):
                continue
            emotions = [Emotions.ANY]
            for i, x in enumerate(d[-10:]):
                if x == "1":
                    emotions.append(emotionOrder[i])
            bits = 0
            for e in emotions:
                bits |= e.value
            dic[term] = emotions
            dicBitmap[term] = bits

def tokenize(text: str) -> Iterator[str]:
    for match in re.finditer(r'\w+', text, re.UNICODE):
        yield match.group(0)

def isWordOfEmotion(word: str, emotion: Emotions) -> bool:
    if word in dic:
        return (dicBitmap[word] & emotion.value) != 0
    return False

def getEmotionsOfWord(word: str) -> List[Emotions]:
    if word not in dic:
        return []
    return dic[word]
    # d = dic[word]
    # for e in Emotions:
    #     if d & e.value != 0:
    #         yield e

def countEmotionsOfWords(words: Iterator[str]) -> Counter[Emotions]:
    c = Counter()
    for w in words:
        c.update(getEmotionsOfWord(w))
    return c

def countEmotionsOfText(text: str) -> Counter[Emotions]:
    return countEmotionsOfWords(tokenize(text))

def separateWordsByEmotion(text: str, dictToUpdate: Dict[Emotions, List[str]] = None) -> Dict[Emotions, List[str]]:
    if dictToUpdate is None:
        dictToUpdate = {}
        for e in Emotions:
            dictToUpdate[e] = []

    words = tokenize(text)
    for w in words:
        if w in dic:
            for e in dic[w]:
                dictToUpdate[e].append(w)

    return dictToUpdate


# -*- coding: utf-8 -*-
"""
Created on Thu Sep  8 11:52:16 2016

@author: rookie
"""

import pandas as pd
import matplotlib as mpl

mpl.style.use('ggplot')

student_dataset = pd.read_csv("students.data",index_col=0)
my_series =student_dataset.G3
my_dataframe = student_dataset[['G3','G2','G1']]

#my_series.plot.hist(alpha=0.5)
my_dataframe.plot.hist(alpha=0.5, normed=False)


import vcf
import argparse
from record import Record

def read_phased_record(inf):
    ps = 0
    update_ps = False
    phased_record = dict()
    for line in inf:
        if line[0] == 'B':
            update_ps = True
        elif line[0] == '*':
            continue
        else:
            buff = line.split()
            pos = int(buff[0])
            hap0 = buff[1]
            hap1 = buff[2]
            if hap0 == '-' or hap1 == '-':
                continue
            if update_ps:
                ps = pos
                update_ps = False
            phased_record[pos] = Record(pos, int(hap0), int(hap1), ps)
    return phased_record

def reset_phasing_info(record: vcf.model._Record):
    gt_str = record.samples[0]['GT']
    new_gt_str = gt_str[0] + '/' + gt_str[2]
    record.samples[0].gt_nums = new_gt_str
    if 'PS' in record.FORMAT.split(':'):
        record.samples[0].data = record.samples[0].data._replace(PS=0)

    record.samples[0].data = record.samples[0].data._replace(GT=new_gt_str)

def update_phasing_info_dchap(in_vcf: vcf.Reader, out_vcf: vcf.Writer, phased_record:dict):
    rec: vcf.model._Record
    count = 0
    for rec in in_vcf:
        count += 1
        pos = rec.POS
        if count in phased_record.keys(): #phased
            record = phased_record[count]
            record.finalize_record(rec)
        else:                           #not phased
            reset_phasing_info(rec)

        out_vcf.write_record(rec)

def update_phasing_info(in_vcf: vcf.Reader, out_vcf: vcf.Writer, phased_record:dict):
    rec: vcf.model._Record
    for rec in in_vcf:
        pos = rec.POS
        if pos in phased_record.keys(): #phased
            record = phased_record[pos]
            record.finalize_record(rec)
        else:                           #not phased
            reset_phasing_info(rec)

        out_vcf.write_record(rec)

def main():
    parser = argparse.ArgumentParser('phaseset_to_vcf.py')
    parser.add_argument('-i', '--inf',  help='in phaseset, refhap format', required=True)
    parser.add_argument('-v', '--vcf', help='input vcf', required=True)
    parser.add_argument('-o', '--out', help='output vcf', required=True)
    parser.add_argument('--dchap', help='dchap format', action="store_true")
    options = parser.parse_args()

    inf = open(file=options.inf, mode="r")
    in_vcf = vcf.Reader(filename=options.vcf)
    outf = vcf.Writer(open(options.out, "w"), template=in_vcf)

    phased_record = read_phased_record(inf)
    if options.dchap:
        update_phasing_info_dchap(in_vcf, outf, phased_record)
    else:
        update_phasing_info(in_vcf, outf, phased_record)

    inf.close()
    outf.close()

    return

if __name__ == '__main__':
    main()


import os
import re
import random
from pytorch_lightning import Trainer
from pytorch_lightning.callbacks import ModelCheckpoint

from nvidia_tao_pytorch.core.callbacks.loggers import TAOStatusLogger
import nvidia_tao_pytorch.core.loggers.api_logging as status_logging
from nvidia_tao_pytorch.core.hydra.hydra_runner import hydra_runner
from nvidia_tao_pytorch.cv.ocrnet.config.default_config import ExperimentConfig
from nvidia_tao_pytorch.cv.ocrnet.model.pl_ocrnet import OCRNetModel

def run_experiment(experiment_spec: ExperimentConfig):
    """run experiment."""
    if experiment_spec.train.results_dir is not None:
        results_dir = experiment_spec.train.results_dir
    else:
        results_dir = os.path.join(experiment_spec.results_dir, "train")
        experiment_spec.train.results_dir = results_dir

    total_epochs = experiment_spec.train.num_epochs
    os.makedirs(f'{results_dir}', exist_ok=True)
    manual_seed = experiment_spec.train.seed
    import torch
    import torch.backends.cudnn as cudnn
    import numpy as np

    random.seed(manual_seed)
    np.random.seed(manual_seed)
    torch.manual_seed(manual_seed)
    torch.cuda.manual_seed(manual_seed)

    cudnn.benchmark = True
    cudnn.deterministic = True

    status_logger_callback = TAOStatusLogger(results_dir,
                                             append=True,
                                             num_epochs=total_epochs)

    status_logging.set_status_logger(status_logger_callback.logger)

    ocrnet_model = OCRNetModel(experiment_spec)
    clip_grad = experiment_spec.train.clip_grad_norm
    gpus_ids = experiment_spec.train.gpu_ids
    num_gpus = experiment_spec.train.num_gpus
    distributed_strategy = None
    if num_gpus > 1 or len(gpus_ids) > 1:
        distributed_strategy = experiment_spec.train.distributed_strategy

    val_inter = experiment_spec.train.validation_interval
    if num_gpus > 1:
        trainer = Trainer(devices=num_gpus,
                          max_epochs=total_epochs,
                          check_val_every_n_epoch=val_inter,
                          default_root_dir=results_dir,
                          enable_checkpointing=False,
                          strategy=distributed_strategy,
                          accelerator='gpu',
                          num_sanity_val_steps=0,
                          gradient_clip_val=clip_grad)
    else:
        trainer = Trainer(gpus=gpus_ids,
                          max_epochs=total_epochs,
                          check_val_every_n_epoch=val_inter,
                          default_root_dir=results_dir,
                          enable_checkpointing=False,
                          strategy=distributed_strategy,
                          accelerator='gpu',
                          num_sanity_val_steps=0,
                          gradient_clip_val=clip_grad)

    ckpt_inter = experiment_spec.train.checkpoint_interval
    ModelCheckpoint.FILE_EXTENSION = ".pth"
    checkpoint_callback = ModelCheckpoint(every_n_epochs=ckpt_inter,
                                          dirpath=results_dir,
                                          save_on_train_epoch_end=True,
                                          monitor=None,
                                          save_top_k=-1,
                                          filename='ocrnet_{epoch:03d}')
    resume_ckpt = experiment_spec['train']['resume_training_checkpoint_path']
    if resume_ckpt:
        status_logging.get_status_logger().write(
            message=f"Resuming training from checkpoint: {resume_ckpt}",
            status_level=status_logging.Status.STARTED
        )
        resumed_epoch = re.search('epoch=(\\d+)', resume_ckpt)
        if resumed_epoch:
            resumed_epoch = int(resumed_epoch.group(1))
        else:
            resumed_epoch = 0
        status_logger_callback.epoch_counter = resumed_epoch + 1  # make sure callback epoch matches resumed epoch

    trainer.callbacks.append(status_logger_callback)
    trainer.callbacks.append(checkpoint_callback)

    trainer.fit(ocrnet_model)

spec_root = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
# Load experiment specification, additially using schema for validation/retrieving the default values.
# --config_path and --config_name will be provided by the entrypoint script.

@hydra_runner(
    config_path=os.path.join(spec_root, "experiment_specs"), config_name="experiment", schema=ExperimentConfig
)
def main(cfg: ExperimentConfig) -> None:
    """Run the training process."""
    try:
        run_experiment(experiment_spec=cfg)
        status_logging.get_status_logger().write(
            status_level=status_logging.Status.SUCCESS,
            message="Training finished successfully."
        )
    except (KeyboardInterrupt, SystemExit):
        status_logging.get_status_logger().write(
            message="Training was interrupted",
            verbosity_level=status_logging.Verbosity.INFO,
            status_level=status_logging.Status.FAILURE
        )
    except Exception as e:
        status_logging.get_status_logger().write(
            message=str(e),
            status_level=status_logging.Status.FAILURE
        )
        raise e

if __name__ == '__main__':
    main()


import numpy as np

import pandas as pd

import matplotlib.pyplot as plt

import seaborn as sns

from sklearn.model_selection import train_test_split
from sklearn.neighbors import KNeighborsClassifier
from sklearn import metrics

data = pd.read_csv('keepers.csv', sep='\t')

data.fillna(value=0, inplace=True)

pd.set_option('display.max_rows', data.shape[0]+1)

#print(data)
#print(data.head())
#print(data.tail())
#print(data.shape)

#sns.pairplot(data, x_vars=feature_cols, y_vars=response_col, size=7, aspect=0.7, kind='reg')
#plt.show()

feature_cols = ['age', 'seasons', 'mins', 'apps', 'minapp', 'starts', 'subs', 'ga', 'ga90', 'sota', 'win', 'draw', 'loss', 'cs', 'csperc', 'saves']
response_col = ['rank']

X = data[feature_cols]
#print(X.shape)
y = data[response_col]
#print(y.shape)

X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=1)

#print('X_train:', X_train)
#print('X_train.shape:', X_train.shape)
#print('y_train:', y_train)
#print('y_train.shape:', y_train.shape)
#print('X_test:', X_test)
#print('y_test:', y_test)
#print('X_train.shape, y_train.shape:', X_train.shape, y_train.shape)
#print('X_test.shape, y_test.shape:', X_test.shape, y_test.shape)
#print('isnan(X_train):', np.isnan(X_train))

k_range = range(1, 26)
scores = []
for k in k_range:
    estimator = KNeighborsClassifier(n_neighbors=k)
    estimator.fit(X_train, y_train.values.ravel())
    y_pred = estimator.predict(X_test)
    scores.append(metrics.accuracy_score(y_test, y_pred))

plt.plot(k_range, scores)
plt.xlabel('k')
plt.ylabel('Accuracy')
plt.show()  # Highest=2 & 24

estimator = KNeighborsClassifier(n_neighbors=24)
estimator.fit(X_train, y_train.values.ravel())

y_pred = estimator.predict(X_test)

print('mean_absolute_error:', metrics.mean_absolute_error(y_test, y_pred))
print('mean_squared_error:', metrics.mean_squared_error(y_test, y_pred))
print('root mean_squared_error:', np.sqrt(metrics.mean_squared_error(y_test, y_pred)))

#print(X_test.iloc[[1]])
#print(y_test.iloc[[1]]) # 29
datum = data.iloc[[88]]
#print(datum)
X = datum[feature_cols]
y_pred = estimator.predict(X)
print('Prediction for 29:', y_pred) # 27

#print(X_test.iloc[[2]])
#print(y_test.iloc[[2]]) # 18
datum = data.iloc[[163]]
#print(datum)
X = datum[feature_cols]
y_pred = estimator.predict(X)
print('Prediction for 18:', y_pred) # 16


from pyshacl import validate
from rdflib import Graph

data_graph = Graph().parse('https://spec.edmcouncil.org/idmp/ontology/master/latest/dev.idmp-quickstart.ttl')
shacl_graph = Graph().parse('https://spec.pistoiaalliance.org/idmp/ontology/AboutIDMPDev-ReferenceIndividuals-SHACL.ttl')

conforms, results_graph, results_text = \
    validate(data_graph,
      shacl_graph=shacl_graph,
      inference='both',
      abort_on_first=False,
      allow_infos=False,
      allow_warnings=False,
      meta_shacl=False,
      advanced=False,
      js=False,
      debug=False)

# for (subject, predicate, object) in results_graph:
#     print(subject, predicate, object)

with open('validation_results.txt', 'w') as results_file:
  results_file.write(results_text)

results_graph.serialize('validation_results.ttl')


from django.http import HttpResponse
from django.views.decorators.csrf import csrf_exempt
import pika
import json
from .models import Bids
from django.core import serializers

def index(request):

    return HttpResponse("Auctions index page.")

@csrf_exempt
def highest_bid(request):
	if request.method == 'GET':
		body_unicode = request.body.decode('utf-8')
		body = json.loads(body_unicode)
		item_id = body["item_id"]
		highest_bid = serializers.serialize('json', Bids.objects.filter(item_id=item_id, highest_bid=True))
		return HttpResponse(highest_bid)

@csrf_exempt
def all_active_bids(request):
	if request.method == 'GET':
		body_unicode = request.body.decode('utf-8')
		body = json.loads(body_unicode)
		account_id = body["account_id"]
		all_bids = serializers.serialize('json', Bids.objects.filter(buyer_account_id=account_id))
		return HttpResponse(all_bids)

@csrf_exempt
def all_highest_bids(request):
	if request.method == 'GET':
		all_bids = serializers.serialize('json', Bids.objects.filter(highest_bid=True))
		return HttpResponse(all_bids)

@csrf_exempt
def all_bids(request):
	if request.method == 'GET':
		all_bids = serializers.serialize('json', Bids.objects.all())
		return HttpResponse(all_bids)


import logging
logging.basicConfig(level=logging.INFO, name=__name__)
import requests
from path import Path
from bs4 import BeautifulSoup

BASE_URL = "http://jtimmer.cts.com"

from argparse import ArgumentParser

def parse_args():
    argparser = ArgumentParser()
    argparser.add_argument('year')
    argparser.add_argument('month')
    argparser.add_argument('--data_dir', default='data/')
    return argparser.parse_args()

def get_urls(base_url):
    soup = BeautifulSoup(requests.get(base_url).text, "html.parser")
    return soup.find_all('a')

def get_csv(data_dir, base_url, csv_url):
    if (data_dir / csv_url).exists():
        logging.info("Loading %s from file." % csv_url)
        with open(data_dir / csv_url) as fp:
            csv_text = fp.read()
        return csv_text
    logging.info("Fetching %s from website." % csv_url)
    csv_text = requests.get(base_url + '/' + csv_url).text.encode('utf-8')
    with open(data_dir / csv_url, 'w') as fp:
        fp.write(csv_text)
    return csv_text

def get_csvs(data_dir, base_url, csv_links):
    csvs = {}
    for link in csv_links:
        csvs[link['href']] = get_csv(data_dir, base_url, link['href'])
    return csvs

def main():
    args = parse_args()

    data_dir = Path(args.data_dir) / args.month / args.year

    data_dir.makedirs_p()

    base_url = "%s/%s/%s%s" % (BASE_URL, args.year, args.month, args.year[-2:])

    links = get_urls(base_url)
    csv_links = [link for link in links if link['href'].split('_')[0] == "yesterday"]
    logging.info("Fetching %u CSVs from %s" % (len(csv_links), base_url))
    csvs = get_csvs(data_dir, base_url, csv_links)

    return csvs

if __name__ == "__main__":
    blah = main()


# -*- coding: utf-8 -*-

"""
Generative Adversarial Nets(GAN)
paper:Generative Adversarial Nets.arxiv:1406.2661v1
This is the implementation of GAN for MNIST dataset
learned from:https://github.com/wiseodd/generative-models/blob/master/GAN/vanilla_gan/gan_tensorflow_v1.py
"""

import tensorflow as tf
import numpy as np
import matplotlib.pyplot as plt
import matplotlib.gridspec as gridspec
import os

from tensorflow.examples.tutorials.mnist import input_data

def xavier_init(size):
    in_dim = size[0]
    xavier_stddev = 1. / tf.sqrt(in_dim /2.)
    return tf.random_normal(shape=size, stddev=xavier_stddev)

def generator(z):
    """
    Generator Net:
        The generator(z) takes 100-dimensional vector and returns 784-dimensional
        vector, which is MNIST image(28*28).z here is the prior for the G(z).In a
        way it learns a mapping between the prior space to Pdata.
    :param z:input noise sampled from N(0,1)
    :return:generated data from noise
    """
    g_w1 = tf.Variable(xavier_init([100, 128]))
    g_b1 = tf.Variable(tf.zeros(shape=[128]))
    g_h1 = tf.nn.relu(tf.matmul(z, g_w1) + g_b1)

    g_w2 = tf.Variable(xavier_init([128, 784]))
    g_b2 = tf.Variable(tf.zeros(shape=[784]))
    g_log_prob = tf.matmul(g_h1, g_w2) + g_b2

    g_prob = tf.nn.sigmoid(g_log_prob)

    theta_g = [g_w1, g_w2, g_b1, g_b2]

    return g_prob, theta_g

d_w1 = tf.Variable(xavier_init([784, 128]))
d_b1 = tf.Variable(tf.zeros(shape=[128]))
d_w2 = tf.Variable(xavier_init([128, 1]))
d_b2 = tf.Variable(tf.zeros(shape=[1]))
theta_d = [d_w1, d_w2, d_b1, d_b2]

def discriminator(x):
    """
    Discriminator Net
    :param x:input data
    :return:is data from x(d_prob) and log likelihood
    """
    d_h1 = tf.nn.relu(tf.matmul(x, d_w1) + d_b1)
    d_logit = tf.matmul(d_h1, d_w2) + d_b2
    d_prob = tf.nn.sigmoid(d_logit)
    return d_prob, d_logit

def sample_z(m,n):
    return np.random.uniform(-1., 1., size=[m, n])

def plot(samples):
    fig = plt.figure(figsize=(4, 4))
    gs = gridspec.GridSpec(4, 4)
    gs.update(wspace=0.05, hspace=0.05)

    for i, sample in enumerate(samples):
        ax = plt.subplot(gs[i])
        plt.axis('off')
        ax.set_xticklabels([])
        ax.set_yticklabels([])
        ax.set_aspect('equal')
        plt.imshow(sample.reshape(28, 28), cmap='Greys_r')

    return fig

def train(batch_size=128,#mini-batch
          learning_rate=0.001#lr
          ):

    print('...... define model ......')
    # define placeholder
    x = tf.placeholder(tf.float32, shape=[None, 784])
    z_dim = 100
    z = tf.placeholder(tf.float32, shape=[None, 100])#z_dim = 100

    g_sample, theta_g = generator(z)
    d_real, d_logit_real = discriminator(x)
    d_fake, d_logit_fake = discriminator(g_sample)

    # D_loss = -tf.reduce_mean(tf.log(D_real) + tf.log(1. - D_fake))
    # G_loss = -tf.reduce_mean(tf.log(D_fake))

    # define loss
    d_loss_real = tf.reduce_mean(tf.nn.sigmoid_cross_entropy_with_logits(logits=d_logit_real, labels=tf.ones_like(d_logit_real)))
    d_loss_fake = tf.reduce_mean(tf.nn.sigmoid_cross_entropy_with_logits(logits=d_logit_fake, labels=tf.zeros_like(d_logit_fake)))
    d_loss = d_loss_real + d_loss_fake

    g_loss = tf.reduce_mean(tf.nn.sigmoid_cross_entropy_with_logits(logits=d_logit_fake, labels=tf.ones_like(d_logit_fake)))

    d_solver = tf.train.AdamOptimizer(learning_rate).minimize(d_loss, var_list=theta_d)
    g_solver = tf.train.AdamOptimizer(learning_rate).minimize(g_loss, var_list=theta_g)

    print('...... loading dataset......')
    mnist = input_data.read_data_sets('MNIST/', one_hot=True)

    if not os.path.exists('GAN_OUT/'):
        os.makedirs('GAN_OUT/')

    print('...... start to training ......')
    sess = tf.Session()
    sess.run(tf.global_variables_initializer())

    index = 0
    for epoch in range(1000000):
        if epoch % 1000 == 0:
            samples = sess.run(g_sample,
                               feed_dict={z: sample_z(16, z_dim)})
            fig = plot(samples)
            plt.savefig('GAN_OUT/{0}.png'.format(str(index).zfill(3)), bbox_inches='tight')
            index = index + 1
            plt.close(fig)

        xs, _ = mnist.train.next_batch(batch_size)

        _, d_loss_curr = sess.run([d_solver, d_loss],
                                  feed_dict={x: xs,
                                             z: sample_z(batch_size, z_dim)})
        _, g_loss_curr = sess.run([g_solver, g_loss],
                                  feed_dict={z: sample_z(batch_size, z_dim)})

        if epoch % 1000 == 0:
            print('---epoch:{0}'.format(epoch))
            print('d_loss:{:.4}'.format(d_loss_curr))
            print('g_loss:{:.4}'.format(g_loss_curr))
            print('---------------------------------')

if __name__ == '__main__':
    train()


import scrapy
import json

class CategoriesSpider(scrapy.Spider):
    name = 'categories'
    allowed_domains = ['www.flipkart.com']
    start_urls = ['https://www.flipkart.com/lc/getData?dataSourceId=websiteNavigationMenuDS_1.0&t=25577905']

    def parse(self, response):
        categories = json.loads(response.body)
        for key in categories['navData']:
        	if 'tabs' in categories['navData'].get(key).keys():
        		for elems in categories['navData'].get(key).get('tabs')[0]['columns']:
        			for i in range(len(elems)):
        				title = elems[i]['title'].encode('utf-8')
        				url = elems[i]['url'].encode('utf-8')

				        yield{
				        	'title': title,
				        	'url': url,
				        }


import re
import requests
import sqlite3
from bs4 import BeautifulSoup
from hudba.models import Genre, Band

def genrescsv():
    genres = Genre.objects.order_by('id')
    url = 'hudba/static/csv/genres.csv'
    f = open(url, 'r')
    lines = f.readlines()
    f.close()
    f = open(url, 'w')
    for number, line in enumerate(lines):
        split = line.split(';')
        correct = 0
        for genre in genres:
            if split[0] == genre.name:
                correct = 1
                break
        if correct == 1:
            f.write(line)
    f.close()

    for genre in genres:
        f = open(url, 'r')
        lines = f.readlines()
        i = 0
        found = 0
        while i < len(lines):
            splitline = lines[i].split(';')
            name = splitline[0]
            if name == genre.name:
                found = 1
                break
            else:
                i += 1
        if found == 0:
            link = "https://cs.wikipedia.org/wiki/" + genre.name
            response = requests.get(url=link)
            soup = BeautifulSoup(response.content, 'html.parser')
            item = soup.select('p')[0]
            para = item.text
            edit = para.replace('\'', ' ')
            finaledit = edit.replace('\"', ' ')
            modify = re.sub(r"\[[^()]*]", "", finaledit)
            csvinput = genre.name + ';' + modify
            f = open(url, 'a', encoding='windows-1250')
            f.write(csvinput)
        f.close()

def bandscsv():
    bands = Band.objects.order_by('id')
    url = 'hudba/static/csv/bands.csv'
    f = open(url, 'r')
    lines = f.readlines()
    f.close()
    f = open(url, 'w')
    for number, line in enumerate(lines):
        split = line.split(';')
        correct = 0
        for band in bands:
            if split[0] == band.title:
                correct = 1
                break
        if correct == 1:
            f.write(line)
    f.close()

    for band in bands:
        f = open(url, 'r')
        lines = f.readlines()
        i = 0
        found = 0
        while i < len(lines):
            splitline = lines[i].split(';')
            name = splitline[0]
            if name == band.title:
                found = 1
                break
            else:
                i += 1
        if found == 0:
            link = "https://cs.wikipedia.org/wiki/" + band.title
            response = requests.get(url=link)
            soup = BeautifulSoup(response.content, 'html.parser')
            csvinput = band.title
            items = [soup.select('p')[0], soup.select('p')[1], soup.select('p')[2]]
            for item in items:
                item = item.text
                item = item.replace('\'', ' ')
                item = item.replace('\"', ' ')
                item = item.replace('¡', ' ')
                modified_item = re.sub(r"\[[^()]*]", "", item)
                csvinput = csvinput + ';' + modified_item
            f = open(url, 'a', encoding='windows-1250')
            f.write(csvinput)
        f.close()

def genressql():
    con = sqlite3.connect('db.sqlite3')
    cur = con.cursor()
    f = open('hudba/static/csv/genres.csv', 'r')
    i = 0
    lines = f.readlines()
    while i < len(lines):
        split = lines[i].split(';')
        article = split[1]
        name = split[0]

        cur.execute("UPDATE hudba_genre SET article = '" + article + "' WHERE name = '" + name + "'")
        i += 1
    con.commit()
    con.close()

def bandssql():
    con = sqlite3.connect('db.sqlite3')
    cur = con.cursor()
    f = open('hudba/static/csv/bands.csv', 'r')
    i = 0
    lines = f.readlines()
    while i < len(lines):
        split = lines[i].split(';')
        article = split[1]
        name = split[0]

        cur.execute("UPDATE hudba_band SET article = '" + article + "' WHERE title = '" + name + "'")
        i += 1

    con.commit()
    con.close()


import re
from collections import defaultdict

txtFile = open('day_15.txt')
lines = txtFile.readlines()

inputSensorMap = defaultdict(lambda: ".")
radarMap = defaultdict(lambda: ".")

minPosX = 9999999999
maxPosX = 0
minPosY = 9999999999
maxPosY = 0

txtOutput = ''

# read input data and get min/max of the x/y coordinates
for line in lines:
    coordinates =  re.findall(r'-?\d+',line)
    sensorPosX, sensorPosY, beaconPosX, beaconPosY = int(coordinates[0]), int(coordinates[1]), int(coordinates[2]), int(coordinates[3])
    inputSensorMap[f"{sensorPosX},{sensorPosY}"] = [int(coordinates[2]), int(coordinates[3]) ]
    radarMap[f"{sensorPosX},{sensorPosY}"] = 'S'
    radarMap[f"{beaconPosX},{beaconPosY}"] = 'B'

    maxPosX = max([maxPosX, sensorPosX, beaconPosX] )
    minPosX = min([minPosX, sensorPosX, beaconPosX] )
    maxPosY = max([maxPosY, sensorPosY, beaconPosY] )
    minPosY = min([minPosY, sensorPosY, beaconPosY] )

#print (inputSensorMap)
#print(minPosX, maxPosX, minPosY, maxPosY)
#print( inputSensorMap.values())

def printMap(minX,maxX, minY, maxY):
    global txtOutput
    global radarMap

    #make x axis labels
    txtOutput = "    "
    for posX in range(minX,maxX+1,1):
         txtOutput += str(posX % 10)
    txtOutput +="\n"

    for posY in range(minY,maxY+1,1):
        #make y-axis labels
        txtOutput +=f'{posY:03d} '
        #make x/y points
        for posX in range(minX,maxX+1,1):
            txtOutput += radarMap[f"{posX},{posY}"]#radarMap[posY][posX]
        txtOutput +="\n"
    print(txtOutput)

def drwRadius(sensX, sensY, mhtRadius):
    newRadius = mhtRadius
    for lvlY in range(0,newRadius,1):
        #upper part
        #right side
        if radarMap[f"{mhtRadius -lvlY+sensX},{sensY-lvlY}"] =='.':
            radarMap[f"{mhtRadius -lvlY+sensX},{(sensY-lvlY)}"] = '#'
        #left side
        if radarMap[f"{sensX-(mhtRadius -lvlY)},{sensY-lvlY}"] =='.':
            radarMap[f"{sensX-(mhtRadius -lvlY)},{(sensY-lvlY)}"] = '#'

        #lower part
        #right side
        if radarMap[f"{(mhtRadius -lvlY)+sensX},{sensY+lvlY}"] =='.':
            radarMap[f"{mhtRadius -lvlY+sensX},{(sensY+lvlY)}"] = '#'
        #left side
        if radarMap[f"{sensX-(mhtRadius -lvlY)},{sensY+lvlY}"] =='.':
            radarMap[f"{sensX-(mhtRadius -lvlY)},{(sensY+lvlY)}"] = '#'

    #peaks of lower and upper part
    if radarMap[f"{sensX},{sensY-newRadius}"] =='.':
        radarMap[f"{sensX},{sensY-newRadius}"]= '#'

    if radarMap[f"{sensX},{sensY+newRadius}"] =='.':
        radarMap[f"{sensX},{sensY+newRadius}"] = '#'

# #Test map view
# for sensorKey, beaconPos in inputSensorMap.items():

#     sensPos = sensorKey.split(",")
#     sensPosX, sensPosY = int(sensPos[0]) , int(sensPos[1])
#     mhtRadius = abs(sensPosX - beaconPos[0]) + abs(sensPosY - beaconPos[1])

#     maxPosX = max([maxPosX, sensPosX+mhtRadius] )
#     minPosX = min([minPosX, sensPosX-mhtRadius] )
#     maxPosY = max([maxPosY, sensPosY+mhtRadius] )
#     minPosY = min([minPosY, sensPosY-mhtRadius] )

#     for radius in range(1,mhtRadius+1,1):
#         drwRadius(sensPosX , sensPosY, radius)
#     #printMap(minPosX, maxPosX, minPosY, maxPosY)

# cntr = 0
# for posX in range(minPosX,maxPosX+1,1):
#     if radarMap[f"{posX},{10}"] == "#":
#         cntr+=1

# printMap(minPosX, maxPosX, minPosY, maxPosY)
# print(cntr)

rangesList = []

def mergeRanges(rngMin, rngMax):

    global rangesList
    tmpRngMax = 0
    tmpRngMin = 0
    fial = False
    if len(rangesList) > 0:
        for rng in rangesList:
            if rngMin <= rng[0] :
                tmpRngMin = rngMin
                if rngMax >= rng[1]:
                    tmpRngMax = rngMax
                    rng[0] = tmpRngMin
                    rng[1] = tmpRngMax
                    fial = False

                elif rngMax <= rng[1] and rngMax >= rng[0]:
                    tmpRngMax = rng[1]
                    rng[0] = tmpRngMin
                    rng[1] = tmpRngMax
                    fial = False
                else:
                    fial = True
            elif rngMin >= rng[0] and rngMin <= rng[1]:
                tmpRngMin = rng[0]
                tmpRngMax = rngMax
                if rngMax < rng[1]:
                    tmpRngMax = rng[1]

                rng[0] = tmpRngMin
                rng[1] = tmpRngMax
                fial = False
            else:
                fial = True

    if len(rangesList) == 0 or fial == True:
        rangesList.append([rngMin, rngMax])

myRanges = []
rngCntr = 0
def mergeRangesByList():

    global rangesList
    global myRanges
    newRangeMerged = False
    tmpRngMin, tmpRngMax = rangesList.pop(0)

    while True:
        newRangeMerged = False

        for rngIdx in range(len(rangesList)):
            if tmpRngMin <= rangesList[rngIdx][0]:
                if tmpRngMax >= rangesList[rngIdx][1]:
                    rangesList.pop(rngIdx)
                    newRangeMerged = True
                    break

                elif tmpRngMax <= rangesList[rngIdx][1] and tmpRngMax >= rangesList[rngIdx][0]:
                    tmpRngMax = rangesList[rngIdx][1]
                    rangesList.pop(rngIdx)
                    newRangeMerged = True
                    break

            elif tmpRngMin >= rangesList[rngIdx][0] and tmpRngMin <= rangesList[rngIdx][1]:
                tmpRngMin = rangesList[rngIdx][0]
                if tmpRngMax < rangesList[rngIdx][1]:
                    tmpRngMax = rangesList[rngIdx][1]

                rangesList.pop(rngIdx)
                newRangeMerged = True
                break

        if newRangeMerged == False: #no connectet range found
            myRanges.append([tmpRngMin, tmpRngMax])
            if len(rangesList) != 0:
                tmpRngMin, tmpRngMax = rangesList.pop(0)
            else:
                break

def getRadiusAtPos(sensX, sensY, mhtRadius, targetPos):

    targetPos = abs(sensY - targetPos) #get absolute distance to the beacon on the y Axis
    lineRange = [(mhtRadius -targetPos+sensX), (sensX-(mhtRadius -targetPos))] #get x position range max to min
    lineRange.sort() #sort elements (min, max)

    #mergeRanges(lineRange[0], lineRange[1])

    return lineRange

lineRangeMinX = 9999999999999999999
lineRangeMaxX = 0
targetLine = 3400528#22 <<-- line with the gap
maxSearchPosY = 4000000

for lineIdx in range(maxSearchPosY): #ca 10 min :(
    targetLine = lineIdx
    myRanges = []
    rangesList = []

    for sensorKey, beaconPos in inputSensorMap.items():
        sensPos = sensorKey.split(",")
        sensPosX, sensPosY = int(sensPos[0]) , int(sensPos[1])
        mhtRadius = abs(sensPosX - beaconPos[0]) + abs(sensPosY - beaconPos[1])

        if abs(sensPosY - targetLine) <= mhtRadius:
            lineRangeX= getRadiusAtPos(sensPosX, sensPosY, mhtRadius, targetLine)
            #print(lineRangeX)
            lineRangeMinX = min( [lineRangeMinX, lineRangeX[0] ] )
            lineRangeMaxX = max( [lineRangeMaxX, lineRangeX[1] ] )
            rangesList.append([lineRangeX[0] , lineRangeX[1]])
            #print(sensorKey, lineRangeX[0] , lineRangeX[1])

    mergeRangesByList()
    if len(myRanges) != 1:
        print(myRanges, lineIdx )
        if  myRanges[0][1] < myRanges[1][1]:
            print("gap at :", myRanges[0][1]+1, lineIdx, ((myRanges[0][1]+1)*maxSearchPosY)+lineIdx )
        else:
            print("gap at :", myRanges[1][1]+1, lineIdx, ((myRanges[1][1]+1)*maxSearchPosY)+lineIdx )

        break

# gtX = 3141837
# gtEerg = 12567351400528 #x * 4000000 + y
# gtY = 12567351400528 -(4000000*gtX) #3400528
# print(gtY)
# if len(myRanges) > 1:
#     if  myRanges[0][1] < myRanges[1][1]:
#         print("gap at :", myRanges[0][1]+1)#(myRanges[1][0] - myRanges[0][1])+myRanges[0][1] )
#     else:
#         print("gap at :", myRanges[1][1]+1)#(myRanges[0][0] - myRanges[1][1])+myRanges[1][1] )

# hh = ''.join(radarMap['11'])
# # print(lineRangeMinX, lineRangeMaxX, hh  )

# # number of al possible positions
# numberPositions = (lineRangeMaxX - lineRangeMinX+1)
# beaconInTargetLn = []

# #subtract beacons from the possible position number
# for sensorKey, beaconPos in inputSensorMap.items():

#     if beaconPos[1] not in beaconInTargetLn:
#         beaconInTargetLn.append( beaconPos[1] )
#         if  beaconPos[1] == targetLine and (beaconPos[0] <= lineRangeMaxX and beaconPos[0] >= lineRangeMinX):
#             numberPositions -=1;

# print(numberPositions, lineRangeMinX, lineRangeMaxX ) #6078701 -1161321 4917380

# #printMap(minPosX, maxPosX, minPosY, maxPosY)


# -*- coding: utf-8 -*-
import copy

import scrapy
from lianjia.items import LianjiaItem

class lianjiaSpider(scrapy.Spider):
    name = 'lianjia'
    allowed_domains = ['cd.fang.lianjia.com']
    host = "https://cd.fang.lianjia.com"
    start_urls = [host + "/loupan/"]
    page_url = "/loupan/pg"
    # 开始页码
    now_page = 1
    # 爬取前100页
    page_last = 100

    def parse(self, response):
        div_list = response.xpath("//div[@class='resblock-desc-wrapper']")
        for div in div_list:
            item = LianjiaItem()
            item['name'] = div.xpath("./div/a/text()").extract_first()
            item['href'] = self.host + div.xpath("./div/a/@href").extract_first()
            item['addr'] = div.xpath("./div[2]/span[1]/text()").extract_first() + "|" + div.xpath(
                "./div[2]/span[2]/text()").extract_first() + "|" + div.xpath("./div[2]/a/text()").extract_first()
            item['base_info'] = div.xpath("./div[1]/span/text()").extract()
            item['apartment'] = div.xpath("./a/span/text()").extract()
            item['area'] = div.xpath("./div[3]/span/text()").extract()
            item['unit_price'] = div.xpath("./div[6]/div[1]/span[1]/text()").extract_first()
            item['total_price'] = div.xpath("/div[6]/div[2]/text()").extract()
            item['advantage'] = div.xpath("./div[5]/span/text()").extract()
            yield scrapy.Request(
                item['href'],
                callback=self.parse_detail,
                meta={'item': copy.deepcopy(item)}
            )
        #     翻页处理
        # 测试返回的网页
        # with open('a.html', 'w', encoding='utf-8') as f:
        #     body = response.body.decode('utf-8')
        #     f.write(body)
        #     当前页 xpath helper 能获取到 但是程序里面获取不到 返回的response 里面不存在 页码 待分析
        # page = response.xpath("//span[@class='active']/text()").extract_first()
        # 最大页码
        # max_page = response.xpath("//span[contains(text(),'...')]/following-sibling::a[1]/text()").extract_first()
        # 下一页
        if self.now_page < self.page_last:
            next_page = self.now_page + 1
            self.now_page = next_page
            next_page_url = self.host + self.page_url + str(next_page)
            print(next_page_url)
            yield scrapy.Request(next_page_url, callback=self.parse)

    def parse_detail(self, response):
        item = response.meta['item']
        item['start_time'] = response.xpath("//p[@class='when']/span[2]/text()").extract_first()
        yield item