import sys
import time

try:
    from unittest2.runner import TextTestRunner
except ImportError:
    from unittest import TextTestRunner

from .result import _XMLTestResult

try:
    # Removed in Python 3
    from cStringIO import StringIO
except ImportError:
    from io import StringIO

class _DelegateIO(object):
    """
    This class defines an object that captures whatever is written to
    a stream or file.
    """

    def __init__(self, delegate):
        self._captured = StringIO()
        self.delegate = delegate

    def write(self, text):
        self._captured.write(text)
        self.delegate.write(text)

    def __getattr__(self, attr):
        try:
            return getattr(self._captured, attr)
        except AttributeError:
            return getattr(self.delegate, attr)

    def reset(self):
        self._captured = StringIO()

class XMLTestRunner(TextTestRunner):
    """
    A test runner class that outputs the results in JUnit like XML files.
    """
    def __init__(self, output='.', outsuffix=None, stream=sys.stderr,
                 descriptions=True, verbosity=1, elapsed_times=True,
                 failfast=False, encoding=None):
        TextTestRunner.__init__(self, stream, descriptions, verbosity,
                                failfast=failfast)
        self.verbosity = verbosity
        self.output = output
        self.encoding = encoding
        if outsuffix:
            self.outsuffix = outsuffix
        else:
            self.outsuffix = time.strftime("%Y%m%d%H%M%S")
        self.elapsed_times = elapsed_times

    def _make_result(self):
        """
        Creates a TestResult object which will be used to store
        information about the executed tests.
        """
        return _XMLTestResult(
            self.stream, self.descriptions, self.verbosity, self.elapsed_times
        )

    def _patch_standard_output(self):
        """
        Replaces stdout and stderr streams with string-based streams
        in order to capture the tests' output.
        """
        sys.stdout = _DelegateIO(sys.stdout)
        sys.stderr = _DelegateIO(sys.stderr)

    def _restore_standard_output(self):
        """
        Restores stdout and stderr streams.
        """
        sys.stdout = sys.stdout.delegate
        sys.stderr = sys.stderr.delegate

    def run(self, test):
        """
        Runs the given test case or test suite.
        """
        try:
            # Prepare the test execution
            self._patch_standard_output()
            result = self._make_result()
            if hasattr(test, 'properties'):
                # junit testsuite properties
                result.properties = test.properties

            # Print a nice header
            self.stream.writeln()
            self.stream.writeln('Running tests...')
            self.stream.writeln(result.separator2)

            # Execute tests
            start_time = time.time()
            test(result)
            stop_time = time.time()
            time_taken = stop_time - start_time

            # Print results
            result.printErrors()
            self.stream.writeln(result.separator2)
            run = result.testsRun
            self.stream.writeln("Ran %d test%s in %.3fs" % (
                run, run != 1 and "s" or "", time_taken)
            )
            self.stream.writeln()

            expectedFails = unexpectedSuccesses = skipped = 0
            try:
                results = map(len, (result.expectedFailures,
                                    result.unexpectedSuccesses,
                                    result.skipped))
            except AttributeError:
                pass
            else:
                expectedFails, unexpectedSuccesses, skipped = results

            # Error traces
            infos = []
            if not result.wasSuccessful():
                self.stream.write("FAILED")
                failed, errored = map(len, (result.failures, result.errors))
                if failed:
                    infos.append("failures={0}".format(failed))
                if errored:
                    infos.append("errors={0}".format(errored))
            else:
                self.stream.write("OK")

            if skipped:
                infos.append("skipped={0}".format(skipped))
            if expectedFails:
                infos.append("expected failures={0}".format(expectedFails))
            if unexpectedSuccesses:
                infos.append("unexpected successes={0}".format(unexpectedSuccesses))

            if infos:
                self.stream.writeln(" ({0})".format(", ".join(infos)))
            else:
                self.stream.write("\n")

            # Generate reports
            self.stream.writeln()
            self.stream.writeln('Generating XML reports...')
            result.generate_reports(self)
        finally:
            self._restore_standard_output()

        return result


"""
AJAX Zipcode example
"""

from time import sleep
from flask import Flask, request

app = Flask(__name__)

@app.route("/getplace", methods=["GET"])
def getplace():
    POSTCODES = {
        "0107": "Oslo",
        "0506": "Oslo",
        "4090": "Hafrsfjord",
        "4033": "Stavanger",
        "4014": "Stavanger",
        "4050": "Sola",
        "4056": "Tananger"
    }
    sleep(1)
    postcode = request.args.get("postcode", None)
    # look up corresponding place or return empty string
    if postcode and (postcode in POSTCODES):
        return POSTCODES[postcode]
    return ""

@app.route("/")
def index():
    return app.send_static_file("zipcode.html")

if __name__ == "__main__":
    app.run(debug=True)


import os
from flask import Flask
from flask_sqlalchemy import SQLAlchemy
from flask_login import LoginManager
from flask_openid import OpenID
from config import basedir, Config
from flask_whooshalchemyplus import index_all
from flask_mail import Mail
from flask_moment import Moment
from flask_babel import Babel

app = Flask(__name__)
app.config.from_object(Config)
db = SQLAlchemy(app)
# 设置存储临时文件
oid = OpenID(app, os.path.join(basedir, 'tmp'))
login_manager = LoginManager()
login_manager.init_app(app)
login_manager.login_view = 'login'
index_all(app)
mail = Mail(app)
moment = Moment(app)
babel = Babel(app)

from app import routes, models, errors
if not app.debug:
    import logging
    from logging.handlers import SMTPHandler, RotatingFileHandler

    # send email
    if app.config['MAIL_SERVER']:
        credentials = None
        if app.config['MAIL_USERNAME'] or app.config['MAIL_PASSWORD']:
            credentials = (app.config['MAIL_USERNAME'], app.config['MAIL_PASSWORD'])
        secure = None
        if app.config['MAIL_USE_TLS']:
            secure = ()
        mail_handler = SMTPHandler(
            mailhost=(app.config['MAIL_SERVER'], app.config['MAIL_PORT']),
            fromaddr='no-reply@' + app.config['MAIL_SERVER'],
            toaddrs=app.config['ADMINS'], subject='Microblog Failure',
            credentials=credentials, secure=secure)
        mail_handler.setLevel(logging.ERROR)
        app.logger.addHandler(mail_handler)

    # 记录错误日志

    if not os.path.exists('logs'):
        os.mkdir('logs')
    file_handler = RotatingFileHandler(os.path.join(basedir, 'logs/microblog.log'), maxBytes=10240, backupCount=10)
    file_handler.setFormatter(logging.Formatter('%(asctime)s %(levelname)s: %(message)s [in %(pathname)s:%(lineno)d]'))
    file_handler.setLevel(logging.INFO)
    app.logger.addHandler(file_handler)

    app.logger.setLevel(logging.INFO)
    # app.logger.info('Microblog startup')

if __name__ == '__name__':
    app.run()


# This is the basic tool
# If code is reused, put it in a function
# Python 3.6.5
# Usage <rulefile> <folder>
import sys
# Import the yara library
import yara
# Import the glob module for recursivly finding files
import glob

"""
def scanHitCallback():
    # What to do when we find a match?
    return
"""

def getFileList(folder):
    globString = folder + "/**/*.php"
    fileList = glob.glob(globString, recursive=True)
    return fileList

def scanFile(compiledRules, fileToScan):
    # Scan a single file
    # Add the callback later

    # Get match
    match = compiledRules.match(fileToScan)
    # Check if it actually got a match
    if(len(match) > 0):
        print("File: %s matches: %s" % (fileToScan, match[0]))
        return True
    else:
        return False

def compileRules(pathToRules):
    rules = yara.compile(pathToRules)
    return rules

def scanner(rules, folder):
    scanList = getFileList(folder)
    print("List of files: %s" % scanList)
    print()
    print("Scanning...")
    scannerMatchedFiles = []
    for f in scanList:
        if(scanFile(rules, f)):
            scannerMatchedFiles.append(f)
    return scannerMatchedFiles

def init(arguments):
    print("--Malware scanner--")
    print()
    # Check if enough arguments are supplied
    # We want at least 3 (script, rules, folder)
    if(len(arguments) == 3):
        rules = compileRules(arguments[1])
        folder = arguments[2]
        # Scan the directory, returns the list of matched files
        matchedFiles = scanner(rules, folder)

        print()
        print("Done Scanning")

        print("-"*10)
        print("Number of matched files: %d" % len(matchedFiles))
        print("All matched files: %s" % matchedFiles)
    else:
        print("Usage: %s %s %s", arguments[0], "<rulefile>", "<folder>")
        exit()

    return

if(__name__ == '__main__'):
    init(sys.argv)
    exit()


from tornado.web import url
from .handlers import PuzzlesListHandler, SinglePuzzleHandler
from .helpers import get_api_url_prefix

def initialize_routes(schema):
    url_prefix = get_api_url_prefix(schema)
    routes = [
        url(url_prefix + r"/", PuzzlesListHandler),
        url(url_prefix + r"/(?P<id>[0-9a-zA-Z]+)", SinglePuzzleHandler),
    ]
    return routes


# uncompyle6 version 3.7.4
# Python bytecode 3.7 (3394)
# Decompiled from: Python 3.6.9 (default, Apr 18 2020, 01:56:04)
# [GCC 8.4.0]
# Embedded file name: /marrow/package/release.py
# Compiled at: 2019-04-29 12:25:50
# Size of source mod 2**32: 610 bytes
"""Release information."""
from collections import namedtuple
version_info = namedtuple('version_info', ('major', 'minor', 'micro', 'releaselevel',
                                           'serial'))(2, 0, 1, 'final', 1)
version = '.'.join([str(i) for i in version_info[:3]]) + (version_info.releaselevel[0] + str(version_info.serial) if version_info.releaselevel != 'final' else '')
author = namedtuple('Author', ['name', 'email'])('Alice Bevan-McGregor', 'alice@gothcandy.com')
description = 'A collection of utilities for resolving object names, names to objects, and managing plugins/extensions.'
url = 'https://github.com/marrow/package/'


#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""beta plane barotropic vorticity model.

This script uses a pseudospectral method to solve the barotropic vorticity
equation in two dimensions

    D/Dt[ω] = 0                                                             (1)

where ω = ξ + f.  ξ is local vorticity ∇ × u and f is global rotation.

Assuming an incompressible two-dimensional flow u = (u, v),
the streamfunction ψ = ∇ × (ψ êz) can be used to give (u,v)

    u = ∂/∂y[ψ]         v = -∂/∂x[ψ]                                        (2)

and therefore local vorticity can be given as a Poisson equation

    ξ = ∆ψ                                                                  (3)

where ∆ is the laplacian operator.  Since ∂/∂t[f] = 0 equation (1) can be
written in terms of the local vorticity

        D/Dt[ξ] + u·∇f = 0
    =>  D/Dt[ξ] = -vβ                                                       (4)

using the beta-plane approximation f = f0 + βy.  This can be written entirely
in terms of the streamfunction and this is the form that will be solved
numerically.

    D/Dt[∆ψ] = +β ∂/∂x[ψ]                                                   (5)

The spectral method defines ψ as a Fourier sum

    ψ = Σ A(t) exp(i (kx + ly))

and as such spatial derivatives can be calculated analytically

    ∂/∂x[ψ] = ikψ       ∂/∂y[ψ] = ilψ

The pseudospectral method will use the analytic derivatives to calculate
values for (u, v) which will then be used to evaluate nonlinear terms.

References:
* This code was developed based on a MATLAB script bvebb.m
  (Original source Dr. James Kent & Prof. John Thuburn)
* And the GFDL documentation for spectral barotropic models
  found here
  [http://www.gfdl.noaa.gov/cms-filesystem-action/user_files/pjp/barotropic.pdf]
* McWilliams Initial Condition inspired by pyqg [https://github.com/pyqg/pyqg]

Downloaded from github.com/jamesp/shallowwater
Modified for DAPPER by Patrick N. Raanes
"""

import matplotlib.pyplot as plt
import numpy as np

from numpy import pi, cos, sin
from numpy.fft import fftshift, fftfreq
from numpy.fft.fftpack import rfft2, irfft2

from tools.utils import Timer
from tools.misc import validate_int

### Configuration
nx = 256
ny = 256                        # numerical resolution
Lx = 1.0
Ly = 1.0                        # domain size [m]
ubar = 0.00                     # background zonal velocity  [m/s]
beta = 8.0                      # beta-plane f = f0 + βy     [1/s 1/m]
n_diss = 2.0                    # Small-scale dissipation of the form ∆^2n_diss,
                                # such that n_diss = 2 would be a ∆^4 hyperviscosity.
tau = 0.3                       # coefficient of dissipation
                                # (smaller => more dissipation)

#Poorly determined coefficients for forcing and dissipation
#r_rayleigh = (1./50000.)/np.sqrt(10.)
r_rayleigh = (1./500.)/np.sqrt(10.)
forcing_amp_factor=100.0/np.sqrt(1.)

### Function Definitions
def ft(phi):
    """Go from physical space to spectral space."""
    return rfft2(phi, axes=(-2, -1))

def ift(psi):
    """Go from spectral space to physical space."""
    return irfft2(psi, axes=(-2,-1))

def courant_number(psix, psiy, dx, dt):
    """Calculate the Courant Number given the velocity field and step size."""
    maxu = np.max(np.abs(psiy))
    maxv = np.max(np.abs(psix))
    maxvel = maxu + maxv
    return maxvel*dt/dx

def grad(phit):
    """Returns the spatial derivatives of a Fourier-transformed variable.
    Returns (∂/∂x[F[φ]], ∂/∂y[F[φ]]) i.e. (ik F[φ], il F[φ])"""
    global ik, il
    phixt = ik*phit        # d/dx F[φ] = ik F[φ]
    phiyt = il*phit        # d/dy F[φ] = il F[φ]
    return (phixt, phiyt)

def velocity(psit):
    """Returns the velocity field (u, v) from F[ψ]."""
    psixt, psiyt = grad(psit)
    psix = ift(psixt)    # v =   ∂/∂x[ψ]
    psiy = ift(psiyt)    # u = - ∂/∂y[ψ]
    return (-psiy, psix)

def spectral_variance(phit):
    global nx, ny
    var_density = 2.0 * np.abs(phit)**2 / (nx*ny)
    var_density[:,0] /= 2
    var_density[:,-1] /= 2
    return var_density.sum()

def anti_alias(phit):
    """Set the coefficients of wavenumbers > k_mask to be zero."""
    k_mask = (8./9.)*(nk+1)**2.
    phit[(np.abs(ksq/(dk*dk)) >= k_mask)] = 0.0

def high_wn_filter(phit):
    """Applies the high wavenumber filter of smith et al 2002"""
    filter_dec = -np.log(1.+2.*pi/nk)/((nk-kcut)**filter_exp)
    filter_idx = np.abs(ksq/(dk*dk)) >= kcut**2.
    phit[filter_idx] *= exp(filter_dec*(np.sqrt(ksq[filter_idx]/(dk*dk))-kcut)**filter_exp)

def forcing_spatial_mask(phi):
   """TODO: Make spatial mask for forcing"""
   phi[idx_sp_mask] *= 0.

_prhs, _pprhs  = 0.0, 0.0  # previous two right hand sides
def adams_bashforth(zt, rhs, dt):
    """Take a single step forward in time using Adams-Bashforth 3."""
    global step, _prhs, _pprhs
    #print("step:",step)
    if step is 0:
        # forward euler
        dt1 = dt
        dt2 = 0.0
        dt3 = 0.0
    elif step is 1:
        # AB2 at step 2
        dt1 = 1.5*dt
        dt2 = -0.5*dt
        dt3 = 0.0
    else:
        # AB3 from step 3 on
        dt1 = 23./12.*dt
        dt2 = -16./12.*dt
        dt3 = 5./12.*dt

    newzt = zt + dt1*rhs + dt2*_prhs + dt3*_pprhs
    _pprhs = _prhs
    _prhs  = rhs
    return newzt

## SETUP

### Physical Domain
dx = Lx / nx
dy = Ly / ny
#dt = 0.4 * 16.0 / nx          # choose an initial dt. This will change
                              # as the simulation progresses to maintain
                              # numerical stability
# Time params
t            = 0.0
tmax         = 10000
dt           = 0.1
PLOT_EVERY_S = 0.2
tplot        = t + PLOT_EVERY_S
FIRST        = True

# Make meshgrid of y coordinates
y = np.linspace(0, Ly, num=ny)
y_arr = np.flipud(np.tile(y,(nx,1)).transpose())

### Spectral Domain
nl = validate_int(ny)
nk = validate_int(nx/2 + 1)
dk = 2.0*pi/Lx
dl = 2.0*pi/Ly
# calculate the wavenumbers [1/m]
# The real FT has half the number of wavenumbers in one direction:
# FT_x[real]    -> complex : 1/2 as many complex numbers needed as real signal
# FT_y[complex] -> complex : After the first transform has been done the signal
# is complex, therefore the transformed domain in second dimension is same size
# as it is in euclidean space.
# Therefore FT[(nx, ny)] -> (nx/2, ny)
# The 2D Inverse transform returns a real-only domain (nx, ny)
k = dk*np.arange(0, nk)     [np.newaxis, :]
l = dl*fftfreq(nl, d=1.0/nl)[:, np.newaxis]

ksq = k**2 + l**2               # squared wavenums (yields laplacian)
ksq[ksq == 0] = 1.0             # avoid divide by zero - set ksq = 1 at zero wavenum
rksq = 1.0 / ksq                # reciprocal 1/(k^2 + l^2)

ik = 1j*k                       # wavenumber mul. imaginary unit is useful
il = 1j*l                       # for calculating derivatives

## Dissipation & Spectral Filters
# Use ∆^2n_diss hyperviscosity to diffuse at small scales
# (i.e. n_diss = 2 would be ∆^4)
# use the x-dimension for reference scale values
nu = ((Lx/(np.floor(nx/3)*2.0*pi))**(2*n_diss))/tau

# High wavenumber filter coefficients.
# Any waves with wavenumber below kcut are not dissipated at all.
filter_exp = 8.
kcut = 30.

# Spectral Filter as per [Arbic and Flierl, 2003]
wvx = np.sqrt((k*dx)**2 + (l*dy)**2)
spectral_filter = exp(-23.6*(wvx-0.65*pi)**4)
spectral_filter[wvx <= 0.65*pi] = 1.0

# ζ and F[ζ] arrays
z  = np.zeros((ny, nx), dtype=np.float64)
zt = np.zeros((nl, nk), dtype=np.complex128)

### Diagnostic arrays
time_arr       = np.zeros(1)
tot_energy_arr = np.zeros(1)

amp = 636

def integrator(xt,t0,dt_tot):

  t = t0

  nsteps = validate_int(dt_tot/dt)
  if nsteps < 10:
    raise Warning('dt_tot not long enough to properly bootstrap\
    multiSTEP (adams-bashford) integrator\
    (and relying solely on Euler is too unstable).')

  global step, _prhs, _pprhs
  step = _prhs = _pprhs = 0

  for k in range(nsteps):
    # calculate derivatives in spectral space
    psit         = -rksq * xt # F[ψ] = - F[ζ] / (k^2 + l^2)
    psixt, psiyt = grad(psit)
    zxt, zyt     = grad(xt)

    # transform back to physical space for pseudospectral part
    z[:] = ift(xt)
    psix = ift(psixt)
    psiy = ift(psiyt)
    zx   = ift(zxt)
    zy   = ift(zyt)

    # Non-linear: calculate the Jacobian in real space
    # and then transform back to spectral space
    jac  = psix * zy - psiy * zx + ubar * zx
    jact = ft(jac)

    # apply forcing in spectral space by exciting certain wavenumbers
    # (could also apply some real-space forcing and then convert
    # into spectral space before adding to rhs)
    rstream     = np.random.RandomState(int(t*10000))
    forcet      = np.zeros_like(ksq, dtype=complex)
    idx         = (14 < np.sqrt(ksq)/dk) & (np.sqrt(ksq)/dk < 20)
    numr        = ksq.shape[0]*ksq.shape[1]
    r_amp       = rstream.rand(numr).reshape(ksq.shape)[idx]
    r_phs       = rstream.rand(numr).reshape(ksq.shape)[idx]
    forcet[idx] = 0.5*amp*(r_amp - 0.5)*exp(1j*2.*pi*r_phs)

    c = 999

    # take a timestep and diffuse
    rhs   = -jact + beta*psixt - xt*r_rayleigh + forcet
    xt[:] = adams_bashforth(xt, rhs, dt)

    #anti_alias
    anti_alias(xt)

    #high_wavenumber_filter
    high_wn_filter(xt)

    t    += dt
    step += 1

    global tplot, time_arr, tot_energy_arr, FIRST
    if t > tplot and __name__ == "__main__":
        with Timer('Diagnostics'):
          urms=np.sqrt(np.mean(psix**2 + psiy**2))
          rhines_scale = np.sqrt(urms/beta)
          tot_energy=0.5*urms**2.
          epsilon = 2 * r_rayleigh * tot_energy
          l_epsilon = (epsilon / beta**3.)**(1./5.)

          time_arr=np.append(time_arr,t)
          tot_energy_arr=np.append(tot_energy_arr,tot_energy)

          force=ift(forcet)

        if FIRST:
          #plt.ion() # plot in realtime
          plt.figure(figsize=(15, 12))
          #plt.clf()
          FIRST = False

          with Timer('vorticity (FIRST)'):
            axv = plt.subplot2grid((2,3), (0, 0), colspan=2, rowspan=2)
            vh = plt.imshow(z, extent=[0, Lx, 0, Ly], cmap=plt.cm.YlGnBu)
            plt.xlabel('x')
            plt.ylabel('y')
            zmax = np.max(np.abs(z))
            plt.clim(-zmax,zmax)
            plt.colorbar(orientation='horizontal')
            plt.title('Rel. vorticity at {:.2f}s dt={:.2f}'.format(t, dt))

          with Timer('Forcing'):
            plt.subplot2grid((2,3), (0, 2))
            fh = plt.imshow(force, extent=[0, Lx, 0, Ly], cmap=plt.cm.YlGnBu)
            plt.xlabel('x')
            plt.ylabel('y')
            forcemax = np.max(np.abs(force))
            plt.clim(-forcemax,forcemax)
            plt.colorbar(orientation='horizontal')
            plt.title('Forcing')

          with Timer('Energy'):
            axl = plt.subplot2grid((2,3), (1, 2))
            lh = plt.plot(time_arr, tot_energy_arr)[0]
            plt.xlabel('Time')
            plt.ylabel('Total Energy')

            plt.subplots_adjust(wspace=0.4,left=0.05,right=0.97)
        else:
          with Timer('Update'):
            vh.set_data(z)
            axv.set_title('Rel. vorticity at {:.2f}s dt={:.2f}'.format(t, dt))
            fh.set_data(force)
            lh.set_data(time_arr, tot_energy_arr)
            axl.set_xlim(right=t)
            #axl.set_ylim((np.min(tot_energy_arr), np.max(tot_energy_arr)))
            axl.set_ylim((0.0000043, 0.00000435))
            axl.ticklabel_format(style='sci',useOffset=4e-6,axis='y')

        plt.pause(0.01)
        print('[{:5d}] {:.2f} Max z: {:2.2f} c={:.2f} dt={:.2f} rh_s={:.3f} l_eps={:.3f} ratio={:2.2f} urms={:2.2f}'.format(
            step, t, np.max(z), c, dt, rhines_scale, l_epsilon, rhines_scale/l_epsilon, urms))
        tplot = t + PLOT_EVERY_S

  return

if __name__ == "__main__":
  z = np.load('mods/Barotropic/z0_1.npz')['z']
  z = z[:ny,:nx]
  zt[:] = ft (z)

  integrator(zt,t,tmax)

### Initial Condition
#   # The McWilliams Initial Condition from [McWilliams - J. Fluid Mech. (1984)]
#   ck   = np.zeros_like(ksq)
#   ck   = np.sqrt(ksq + (1.0 + (ksq/36.0)**2))**-1
#   piit = rand(*ksq.shape)*ck + 1j*rand(*ksq.shape)*ck
#
#   pii  = ift(piit)
#   pii  = pii - pii.mean()
#   piit = ft(pii)
#   KE   = spectral_variance(piit*np.sqrt(ksq)*spectral_filter)
#
#   qit  = -ksq * piit / np.sqrt(KE)
#   qi   = ift(qit)
#   z[:] = qi
#
#   # calc a reasonable forcing amplitude
#   amp = forcing_amp_factor * np.max(np.abs(qi))
#
#   # initialise the transformed ζ
#   zt[:] = ft (z)
#   anti_alias (zt)
#   z[:]  = ift(zt)
#
#   # initialise the storage arrays
#   time_arr[0]=t
#
#   psit = -rksq * zt         # F[ψ] = - F[ζ] / (k^2 + l^2)
#   psixt, psiyt = grad(psit)
#   psix = ift(psixt)
#   psiy = ift(psiyt)
#
#   urms              = np.sqrt(np.mean(psix**2 + psiy**2))
#   tot_energy        = 0.5*urms**2.
#   tot_energy_arr[0] = tot_energy


# -*- coding: utf-8 -*-
"""
Created on Tue Apr 21 14:31:58 2020

@author: ninjaac
"""
from keras.datasets import mnist
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt

(x_train,y_train), (x_test,y_test)=mnist.load_data()
some_digit=x_train[0]
#plt.imshow(some_digit,cmap = plt.cm.binary)
#plt.show()
y_train_5=(y_train==5)
print(np.shape(x_train))
x_train=x_train.reshape((60000,784))
print(np.shape(x_train))
# select a classifier and fit the model
from sklearn.linear_model import SGDClassifier
sgd=SGDClassifier(random_state=42)
sgd.fit(x_train,y_train_5)

#model performance mesure
from sklearn.model_selection import cross_val_score
score=cross_val_score(sgd,x_train,y_train_5,cv=5,scoring='accuracy')
print(score)

# more better prediction will be found by presicion and recall score
from sklearn.model_selection import cross_val_predict
y_train_pred = cross_val_predict(sgd, x_train, y_train_5, cv=3)
from sklearn.metrics  import precision_score,recall_score,f1_score
presicion=precision_score(y_train_5,y_train_pred)
print(f"precision {presicion}")
recall_sc=recall_score(y_train_5,y_train_pred)
print(f"Recall value {recall_sc}")
f1=f1_score(y_train_5,y_train_pred)
print(f" fi score value {f1}")
#presicion recall curve
y_scores = cross_val_predict(sgd, X_train, y_train_5, cv=3,
                             method="decision_function")
from sklearn.metrics import precision_recall_curve
precisions,recalls,thresholds=precision_recall_curve(y_train_5,y_scores)
def plot_precision_recall_vs_threshold(precisions, recalls, thresholds):
    plt.plot(thresholds, precisions[:-1], "b--", label="Precision")
    plt.plot(thresholds, recalls[:-1], "g-", label="Recall")
    plt.legend()
    plt.show()

plot_precision_recall_vs_threshold(precisions, recalls, thresholds)


import numba as nb
from numba import prange
from models.AHNFunctions import CreateLinearCompound
from models.AHNFunctions import DataInMolecules
from models.AHNFunctions import ComputeMoleculeParameters
import numpy as np

def PiranhaFeastOptimizer(D, NP, R, b, c, nFES, Lower, Upper, x, y, n):
    #Initialize parameters
    R_matrix = np.zeros((R,D))
    R_score = np.full((R), float("inf"))
    R_rad = np.full((R), 0.10)
    V = np.random.rand(NP,D)
    eval_flag = True  # evaluations flag
    evaluations = 0  # evaluations counter
    P = np.zeros((NP, D))
    P_score = np.zeros((NP, 1))

    C = CreateLinearCompound(n)
    def Fun(D, sol, x, y, C):
        #@nb.jit(nopython=True, parallel=True)
        def evaluate(D, sol, x, y, C):  # sol is the possible solution
            # Reshape the particle for matching the prototype structure
            sol = np.asarray(sol).reshape((n, x.shape[1]))
            # Split training data over molecules
            SigmaSplitX, SigmaSplitY = DataInMolecules(x, y, sol)
            # initialize procedure to fit molecular parameters
            error = np.zeros((n + 1, np.size(y, axis=1)))
            omega = C
            H = [[0] for i in range(n)]
            # Find molecular parameters
            for j in list(range(0, n)):
                Xi = SigmaSplitX[j]
                Yi = SigmaSplitY[j]
                if Xi.any():
                    [H[j], error[j][:]] = ComputeMoleculeParameters(Xi, Yi, int(omega[j]))
            #ahn = {'H': H, 'Pi': sol, 'n': n, 'C': C}
            fitness = np.sum(error)
            return fitness
        fitness = evaluate(D=D, sol=sol, x=x, y=y, C=C)
        return fitness

    #@nb.jit(nopython=True, parallel=False)
    def initialization(D, NP, Lower, Upper, P):
        #P = np.zeros((NP, D))
        #P_score = np.zeros((NP, 1))
        """Initialize positions."""
        for i in range(0, NP):
            for j in range(0, D):
                P[i, j] = np.random.rand() * (Upper[j] - Lower[j]) + Lower[j]
        return P

    #@nb.jit(nopython=True, parallel=False)
    def eval_true(evaluations, nFES, eval_flag):
        """Check evaluations."""
        if evaluations == nFES:
            eval_flag = False
        return eval_flag

    #@nb.jit(nopython=True, parallel=False)
    def bounds(D, Lower, Upper, position):
        """Keep it within bounds."""
        for i in range(D):
            if position[i] < Lower[i]:
                position[i] = Lower[i]
            if position[i] > Upper[i]:
                position[i] = Upper[i]
        return position

    """
    def attack(R, P, b, V):
        V_actual = V + np.random.rand() * b * (R - P)
        P_actual = P + V_actual
        return V_actual, P_actual

    def navigate(c, d, R):
        t=np.random.uniform(-1,1)
        M = d*np.exp(c*t)*np.cos(2*np.pi*t) + R
        return M
    """
    @nb.jit(nopython=True, parallel=True)
    def actual_pos(NP, P, V, R_matrix, R_rad, c, Fit, R_score):
        for i in prange(NP):
            #dist_matrix = Fit[i] - R_score #np.asarray([Fit - R_score[i] for i in range(len(R_matrix))])
            dist_matrix = np.asarray([np.linalg.norm(P[i] - R_matrix[j]) for j in prange(len(R_matrix))])  # Distancia entre pirañas y presas
            R_ind = np.argmin(dist_matrix)  # Identifica la presa mas cercana a la piraña i
            if (dist_matrix[R_ind] < R_rad[R_ind]) and (dist_matrix[R_ind] != 0):
                V[i] = V[i] + np.random.rand()*b*(R_matrix[R_ind] - P[i])
                P[i] = P[i] + V[i]
                #V[i], P[i] = attack(R=R_matrix[R_ind], P=P[i], b=b, V=V[i])
                R_rad[R_ind] = R_rad[R_ind] * (0.90 ** (R_ind + 1))
            else:
                acum = np.sum(dist_matrix)
                Prob_nj = np.divide(acum, dist_matrix)#np.asarray([(acum / d) for d in dist_matrix])
                acum = np.sum(Prob_nj)
                Prob_nj = np.divide(Prob_nj, acum)#Prob_nj/acum
                #Prob_nj = [w / acum for w in Prob_nj]
                #selectR = np.random.choice(a=list(range(len(R_matrix))), p=Prob_nj)
                selectR = np.searchsorted(np.cumsum(Prob_nj), np.random.rand(1))[0]
                t = np.random.uniform(-1,1)
                P[i] = dist_matrix[selectR] * np.exp(c*t)*np.cos(2*np.pi*t) + R_matrix[selectR]
                #P[i] = navigate(c=c, d=dist_matrix[selectR], R=R_matrix[selectR])

        return P, V
    """
    #@nb.jit(nopython=True, parallel=True)
    def actual_pos2(NP, P, V, R_matrix, R_rad, c, Fit, R_score):
        for i in range(NP):
            R_ind = 0
            for j in prange(len(R_score)):
                if (Fit[i] < np.multiply(R_score[j],1+R_rad[j])):
                    R_ind = j
                    break

            #dist_matrix = Fit[i] - R_score #np.asarray([Fit - R_score[i] for i in range(len(R_matrix))])
            #dist_matrix = np.asarray([np.linalg.norm(P[i] - R_matrix[j]) for j in range(len(R_matrix))])  # Distancia entre pirañas y presas
            #R_ind = np.argmax(dist_matrix)  # Identifica la presa mas cercana a la piraña i
            if R_ind != 0:
                V[i] = V[i] + np.random.rand()*b*(R_matrix[R_ind] - P[i])
                P[i] = P[i] + V[i]
                #V[i], P[i] = attack(R=R_matrix[R_ind], P=P[i], b=b, V=V[i])
                R_rad[R_ind] = (R_rad[R_ind]) * (0.90 ** (R_ind + 1))
            else:
                dist_matrix = np.asarray([np.linalg.norm(P[i] - R_matrix[j]) for j in range(len(R_matrix))])
                acum = np.sum(dist_matrix)
                Prob_nj = np.divide(acum, dist_matrix)#np.asarray([(acum / d) for d in dist_matrix])
                acum = np.sum(Prob_nj)
                Prob_nj = np.divide(Prob_nj, acum)#Prob_nj/acum
                #Prob_nj = [w / acum for w in Prob_nj]
                #selectR = np.random.choice(a=list(range(len(R_matrix))), p=Prob_nj)
                selectR = np.searchsorted(np.cumsum(Prob_nj), np.random.rand(1))[0]
                t = np.random.uniform(-1,1)
                P[i] = dist_matrix[selectR] * np.exp(c*t)*np.cos(2*np.pi*t) + R_matrix[selectR]
                #P[i] = navigate(c=c, d=dist_matrix[selectR], R=R_matrix[selectR])

        return P, V
    """
    def move(R_matrix, R_score, R_rad, V, eval_flag, evaluations, c, P, P_score):
        P = initialization(D=D, NP=NP, Lower=Lower, Upper=Upper, P=P)   #Inicialización
        #P_score[i] = Fun(D=D, sol=P[i], x=x, y=y, C=C)
        while eval_flag is not False:
            for i in range(NP):
                P[i] = bounds(D=D, Lower=Lower, Upper=Upper, position=P[i]) #Valida limites
                eval_flag = eval_true(evaluations=evaluations, nFES=nFES, eval_flag=eval_flag)
                if eval_flag is not True:
                    break
                Fit = Fun(D=D, sol=P[i], x=x, y=y, C=C) #Fitness de la solucion
                P_score[i] = Fit
                evaluations = evaluations + 1
                ####    Identifica presas
                for z in range(len(R_matrix)):
                    if Fit <= R_score[z]:
                        R_score[z] = Fit
                        R_matrix[z] = P[i]
                        R_rad[z] = 0.10
                        break
                #### Si es la primera evaluacion, ordenar R
                if evaluations == (len(R_matrix)):
                    ind = np.argsort(R_score)
                    R_score = R_score[ind]
                    R_matrix = R_matrix[ind]
                    R_rad = R_rad[ind]
                """
                dist_matrix = np.asanyarray([np.linalg.norm(P[i] - R_matrix[j]) for j in range(len(R_matrix))])   #Distancia entre pirañas y presas
                R_ind = np.argmin(dist_matrix) # Identifica la presa mas cercana a la piraña i
                if dist_matrix[R_ind] < R_rad[R_ind]:
                    V[i], P[i] = attack(R=R_matrix[R_ind], P=P[i], b=b, V=V[i])
                    R_rad[R_ind] = R_rad[R_ind]*0.95**(R_ind+1)

                else:
                    acum = sum(dist_matrix)
                    Prob_nj = np.asanyarray([(acum/d) for d in dist_matrix])
                    acum = sum(Prob_nj)
                    Prob_nj = [w / acum for w in Prob_nj]
                    selectR = np.random.choice(a=list(range(len(R_matrix))), p=Prob_nj)
                    P[i] = navigate(c=c, d=dist_matrix[selectR], R=R_matrix[selectR])
                """
            P, V = actual_pos(NP=NP, P=P, V=V, R_matrix=R_matrix, R_rad=R_rad, c=c, Fit=P_score, R_score=R_score)
        Best_pray = R_score[0]

        return Best_pray

    def run(R_matrix, R_score, R_rad, V, eval_flag, evaluations, c, P, P_score):
        return move(R_matrix, R_score, R_rad, V, eval_flag, evaluations, c, P, P_score)

    Best_pray = run(R_matrix, R_score, R_rad, V, eval_flag, evaluations, c, P, P_score)

    return Best_pray


#!/usr/bin/python

import xml.etree.ElementTree as xml
import random

team = [("president", 1)]
vices = random.randint(4,7)
v=0
while v < vices:
    v+=1
    team.append(('vice president %s' % v, 2 , v) )
    members = random.randint(2,10)
    me=0
    while me < members:
        me+=1
        team.append(('team member %s' % me , 3, v))
print(team)

model = xml.Element('mxGraphModel')
root = xml.Element('root')
model.append(root)
m=0
zero = xml.Element('mxCell',id='0')
one = xml.Element('mxCell',id='1', parent='0')
root.append(zero)
root.append(one)

for member in team:
    if member[1] == 1:
        newCell=xml.Element('mxCell', id=member[0]+'_vertex', value=member[0], vertex='1', parent='1')
        newMxGeometry=xml.Element('mxGeometry', x='0', y='0', width='80', height='70')
        newMxGeometry.set('as','geometry')
        newCell.append(newMxGeometry)

    elif member[1] == 2:
        newCell=xml.Element('mxCell', id=member[0]+'_vertex', value=member[0], vertex='1', parent='1')
        newMxGeometry=xml.Element('mxGeometry', x='0', y='0', width='70', height='60')
        newMxGeometry.set('as','geometry')
        newCell.append(newMxGeometry)
        newEdge=xml.Element('mxCell', id=member[0]+"_edge",
                            value='', edge="1", parent="1", source='president_vertex', target=member[0]+'_vertex')
        newEdgeGeo=xml.Element('mxGeometry', relative='1')
        newEdgeGeo.set('as','geometry')
        newEdge.append(newEdgeGeo)
        root.append(newEdge)
    elif member[1] == 3:
        m+=1
        newCell=xml.Element('mxCell', id=member[0]+'-'+str(m)+'_vertex', value=member[0],
                            vertex='1', parent='1')
        newMxGeometry=xml.Element('mxGeometry', x='0', y='0', width='60', height='50')
        newMxGeometry.set('as','geometry')
        newCell.append(newMxGeometry)
        newEdge=xml.Element('mxCell', id=member[0]+'-'+str(m)
                            +'_edge', value='', edge="1", parent="1",
                            target=member[0]+'-'+str(m)+'_vertex', source='vice president '+str(member[2])+'_vertex')
        newEdgeGeo=xml.Element('mxGeometry', relative='1')
        newEdgeGeo.set('as','geometry')
        newEdge.append(newEdgeGeo)
        root.append(newEdge)
    root.append(newCell)

file = open("example.xml", 'w')
xml.ElementTree(model).write(file)
file.close()


#!/usr/bin/python
# -*- coding: utf-8 -*-
from __future__ import print_function
import argparse
from collections import defaultdict
import json
import logging
import os
import subprocess
import sys

from qald.sparql_utils import Query, QueryManager, GetQueriesResults
from utils.evaluation import compute_f1_score
from utils.priority_queue import PriorityQueue
from utils.tree_tools import tree_or_string, get_top, IsString
from utils.tools_generic_transducer import TiburonToStanford

parser = argparse.ArgumentParser()
parser.add_argument(
  "infile", nargs='?', type=argparse.FileType('r'), default=sys.stdin)
parser.add_argument(
  "--kgen", nargs='?', type=int, default="10",
  help="Maximum number of queries to be generated.")
parser.add_argument(
  "--cores", nargs='?', type=int, default="1",
  help="Number of cores for multiprocessing.")
parser.add_argument(
  "--kout", nargs='?', type=int, default="10",
  help="N-best maximum number of trees (queries) to output as result.")
args = parser.parse_args()

logging.basicConfig(level=logging.DEBUG)

TIBURON = '../tiburon/tiburon'
temp_fname = 'temp.wrtg'
temp_fname_err = 'temp.wrtg.err'

def rename_vars(tree, var_index=0, var_pref = '?p'):
  if IsString(tree):
    return tree
  for p in tree.treepositions('leaves'):
    if '[]' in tree[p]:
      reverse_op = '!' if tree[p].startswith('!') else ''
      tree[p] = reverse_op + var_pref + str(var_index)
      var_index += 1
  return tree

def is_plausible_tree(tree):
  """
  Returns True if there are more than one predicate variables.
  """
  if IsString(tree):
    return False
  preds = set(l for l in tree.leaves() if l.lstrip('!').startswith('?p'))
  return len(preds) < 2

def filterout_trees(trees):
  return (tree for tree in trees if is_plausible_tree(tree))

def generate_trees(grammar, kgen):
  with open(temp_fname, 'w') as fout:
    for p in grammar:
      fout.write(p + '\n')
  # with open(os.devnull, 'w') as FNULL:
  with open(temp_fname_err, 'w') as FNULL:
    output = subprocess.check_output(
      [TIBURON,
       temp_fname,
       '--semiring', 'real',
       '-n',
       '-k', str(kgen)],
      stderr=FNULL)
  trees = [tl.split(' # ')[0] for tl in output.split('\n') if tl.strip()]
  trees = [tree_or_string(TiburonToStanford(t)) for t in trees]
  trees = [rename_vars(t) for t in trees]
  trees = filterout_trees(trees)
  return trees

def normalize_answer(answer):
  """
  Normalize answers from Freebase. At the moment,
  only URIs are shortened with the prefix "fb:".
  """
  return answer.replace('http://rdf.freebase.com/ns/', 'fb:')

def integrate_results(query, results):
  """
  results is a list of results:
  [[answer, var1 inst, var2 inst, ...], ...]
  This function integrates all answers for each
  combination of variable instantiations.
  """
  integration = defaultdict(list)
  for result in results:
    answer = normalize_answer(result[0])
    integration[tuple(result[1:])].append(answer)
  d = {'ldcsc' : str(query.ldcsc)}
  d['results'] = []
  for instantiations, answers in integration.items():
    var_inst = dict(zip(query.query_vars, instantiations))
    d['results'].append({'instantiations' : var_inst, 'answers' : answers})
  return d

def execute_trees(trees, query_manager, ncores=1):
  queries = [Query.fromldcsc(tree, ['?p']) for tree in trees]
  queries_results = GetQueriesResults(queries, query_manager, ncores)
  exec_results = []
  for query, results in zip(queries, queries_results):
    exec_result = integrate_results(query, results)
    exec_results.append(exec_result)
  return exec_results

def get_best_execs(execs, gold_answers, kout=10):
  """
  Given query executions (which are dictionaries with the following entries):
  ldcsc: constituent representation of l-DCS tree.
  instantiations: a map from variable name to predicate/entity URIs.
  answers: a list of entities (which are the answer retrieved).
  It returns the kout best executions.
  gold_answers is a set with the gold answers for the question.
  """
  queue = PriorityQueue(kout, reverse=True)
  for execution in execs:
    if execution:
      for result in execution['results']:
        answers = set(result['answers'])
        f1_score = compute_f1_score(answers, gold_answers)
        exec_instantiation = {
          'ldcsc' : execution['ldcsc'],
          'instantiations' : result['instantiations'],
          'answers' : result['answers']}
        queue.Push(f1_score, exec_instantiation)
  scored_execs = queue.GetSortedScoresAndItems()
  execs = []
  for score, execution in scored_execs:
    execution['score'] = score
    execs.append(execution)
  return execs

def index_grammar(grammar):
  """
  Creates a dictionary where:
    keys are RHSs of productions,
    values are tuples ([production], weight), that is,
      a tuple where the first element is a list of productions,
      and the second element is a weight (float).
  """
  index = dict()
  for production in grammar:
    if '->' not in production:
      continue
    lhs, rhs_weight = production.split(' -> ')
    if '#' in rhs_weight:
      rhs, weight_str = rhs_weight.split(' # ')
      weight = float(weight_str)
    else:
      rhs = rhs_weight
      weight = 1.0
    rhs_stan = TiburonToStanford(rhs)
    if rhs in index:
      index[rhs_stan][0].append('{0} -> {1}'.format(lhs, rhs))
      index[rhs_stan][1] = max(index[rhs_stan][1], weight)
    else:
      index[rhs_stan] = [['{0} -> {1}'.format(lhs, rhs)], weight]
  return index

def get_wprods_from_grammar_index(grammar_index):
  wprods = ['start']
  for rhs, (prods, weight) in grammar_index.items():
    for prod in prods:
      wprods.append('{0} # {1}'.format(prod, weight))
  return wprods

def estimate_wrtg_weights(
    grammar, gold_answers, query_manager, cores=1, batch_size=10):
  grammar_rhs_index = index_grammar(grammar)
  trees = list(generate_trees(grammar, batch_size))
  execs = execute_trees(trees, query_manager, cores)
  for tree, ex in zip(trees, execs):
    productions = grammar_rhs_index.get(str(tree), None)
    if productions is not None:
      if ex and ex['results']:
        for result in ex['results']:
          answers = set(result['answers'])
          f1_score = compute_f1_score(answers, gold_answers)
          productions[-1] = f1_score
      else:
        productions[-1] = 0.0
  grammar = get_wprods_from_grammar_index(grammar_rhs_index)
  return grammar

num_restarts = 0
batch_size = 1000

score = 0.0
total_examples = 0
with QueryManager() as query_manager:
  for example_str in args.infile:
    total_examples += 1
    example = json.loads(example_str.strip())
    gold_answers = set(example['answer_entities'])
    grammar = example.get('grammar', None)
    if grammar is None:
      print('x', end='', file=sys.stderr)
      continue
    # from pudb import set_trace; set_trace()
    for i in range(num_restarts):
      logging.info('Restart {0}'.format(i))
      grammar = estimate_wrtg_weights(
        grammar, gold_answers, query_manager, args.cores, batch_size)
    trees = generate_trees(grammar, args.kgen)
    execs = execute_trees(trees, query_manager, args.cores)
    best_execs = get_best_execs(execs, gold_answers, args.kout)
    example['executions'] = best_execs
    example['grammar'] = grammar
    if best_execs:
      score += best_execs[0]['score']
    print(json.dumps(example))
    print('.', end='', file=sys.stderr)

print('Averaged F1-score: {0}'.format(score / total_examples), file=sys.stderr)


from django.template import loader, Context,RequestContext
from django.http import HttpResponse
from django.shortcuts import render_to_response, get_object_or_404,redirect
from Deanery.models import Groups, Students
from django.shortcuts import render
from Deanery.forms import StudForm, GroupForm
from django.http import HttpResponseRedirect
from django.contrib.auth.decorators import login_required

@login_required
def archive(request):
    ls_groups = Groups.objects.all()
    data = dict(ls_groups=ls_groups)
    return render_to_response('tm_group.html',
        data,
        context_instance=RequestContext(request))

#    ls_groups = Groups.objects.all()
#    t=loader.get_template("tm_group.html")
#    c=Context({'ls_groups':ls_groups})
#    return HttpResponse(t.render(c))

@login_required
def students_in_this_group(request,group_id):
    current_group = get_object_or_404(Groups, id=group_id)
    variables = RequestContext(request, {
        'current_group': current_group,
        })
    return render_to_response(
        'tm_students.html',
        variables
    )

@login_required
def group_add(request):
    group = Groups()
    if request.method == 'POST':
        form = GroupForm(data=request.POST, instance=group)
        if form.is_valid():
            form.save()
            return HttpResponseRedirect('/Deanery/')
    else:
        form = GroupForm(instance=group)
        variables = RequestContext(request, {
        'type': 'Deanery',
        'form': form,
        })
    return render_to_response(
        'tm_group_add.html',
        variables
    )

@login_required
def group_delete(request, group_id):
    group = get_object_or_404(Groups, id=group_id)
    for Students in group.rel_students.all():
        Students.group = None
        Students.save()
    group.delete()
    return HttpResponseRedirect('/Deanery/')

@login_required
def group_edit(request, group_id):
    group = get_object_or_404(Groups, id=group_id)
    if request.method == 'POST':
        form = GroupForm(data=request.POST, instance=group)
        if form.is_valid():
            form.save()
            return HttpResponseRedirect('/Deanery/')
    else:
        form = GroupForm(instance=group)
    variables = RequestContext(request, {
        'obj': group,
        'type': 'Deanery',
        'form': form,
        })
    return render_to_response(
        'tm_group_edit.html',
        variables
    )

@login_required
def student_add(request):
    std = Students()
    if request.method == 'POST':
        form = StudForm(data=request.POST, instance=std)
        if form.is_valid():
            form.save()
            return HttpResponseRedirect(std.get_absolute_url())
    else:
        form = StudForm(instance=std)
    variables = RequestContext(request, {
        'type': 'Deanery',
        'form': form,
        })
    return render_to_response(
        'tm_group_add.html',
        variables
    )

@login_required
def student_view(request, student_id):
    std = get_object_or_404(Students, id=student_id)
    variables = RequestContext(request, {
        'student': std,
        })
    return render_to_response(
        'tm_std_view.html',
        variables
    )

@login_required
def student_edit(request, student_id):
    std = get_object_or_404(Students, id=student_id)
    if request.method == 'POST':
        form = StudForm(data=request.POST, instance=std)
        if form.is_valid():
            form.save()
            return HttpResponseRedirect(std.get_absolute_url())
    else:
        form = StudForm(instance=std)
        variables = RequestContext(request, {
        'obj': std,
        'type': 'Deanery',
        'form': form,
        })
    return render_to_response(
        'tm_group_edit.html',
        variables
    )

@login_required
def student_delete(request, student_id):
    std = get_object_or_404(Students, id=student_id)
    group_id = std.groups.id
    std.delete()
    return HttpResponseRedirect('/Deanery/students-in-this-group/%s/' % group_id)


__author__ = 'Vladimir Bolshakov <vovanbo@gmail.com>'

import json
import copy
from collections import OrderedDict
from typing import Callable, List, MutableMapping, Optional, Union

import ttree.utils
from ttree.common import ASCIIMode, TraversalMode
from ttree.exceptions import (
    NodeNotFound, MultipleRoots, DuplicatedNode, LinkPastRootNode, LoopError
)
from .node import Node

class Tree(OrderedDict):
    """
    The Tree object defines the tree-like structure based on
    :class:`Node` objects. A new tree can be created from scratch without any
    parameter or a shallow/deep copy of another tree. When ``deepcopy=True``,
    a deepcopy operation is performed on feeding ``tree`` parameter and
    *more memory is required to create the tree*.
    """
    def __init__(self, tree: 'Tree' = None, deepcopy: bool = False):
        """Initiate a new tree or copy another tree with a shallow or
        deepcopy copy.
        """
        super(Tree, self).__init__()

        #: id of the root node
        self.root = None

        if tree is not None:
            if not isinstance(tree, Tree):
                raise TypeError('Tree instance is required.')

            self.root = tree.root
            self.__merge_tree(tree, deepcopy)

    def __str__(self) -> str:
        return ttree.utils.print_tree(self, ascii_mode='simple')

    def __getitem__(self, item):
        try:
            return super(Tree, self).__getitem__(item)
        except KeyError:
            raise NodeNotFound(f"Node '{item}' is not in the tree")

    def __setitem__(self, key, value, **kwargs):
        super(Tree, self).__setitem__(key, value, **kwargs)
        if isinstance(value, Node):
            value._tree = self

    def __delitem__(self, key, **kwargs):
        if key in self:
            self[key]._tree = None
        super(Tree, self).__delitem__(key, **kwargs)

    def __merge_tree(self, other: 'Tree', deepcopy: bool = False):
        if deepcopy:
            for node_id in other:
                self[node_id] = copy.deepcopy(other[node_id])
        else:
            self.update(other)

    @property
    def paths_to_leaves(self):
        """
        Use this property to get the identifiers allowing to go from the root
        nodes to each leaf.
        Return a list of list of identifiers, root being not omitted.

        For example:

        .. code-block:: text

            Harry
            |___ Bill
            |___ Jane
            |    |___ Diane
            |         |___ George
            |              |___ Jill
            |         |___ Mary
            |    |___ Mark

        Result:

        .. code-block:: python3

            [['harry', 'jane', 'diane', 'mary'],
             ['harry', 'jane', 'mark'],
             ['harry', 'jane', 'diane', 'george', 'jill'],
             ['harry', 'bill']]
        """
        return [[n for n in self.rsearch(l.id)][::-1] for l in self.leaves()]

    def add_node(self, node: Node, parent: Node = None):
        """
        Add a new node to tree.

        Add a new node object to the tree and make the parent as the root
        by default.
        """
        if not isinstance(node, Node):
            raise TypeError('First parameter must be instance of Node.')

        if node.id in self:
            raise DuplicatedNode(f"Node with ID '{node.id}' "
                                 f"is already exists in tree.")

        pid = parent.id if isinstance(parent, Node) else parent

        if pid is None:
            if self.root is not None:
                raise MultipleRoots('A tree takes one root merely.')

            self.root = node.id
        elif pid not in self:
            raise NodeNotFound(f"Parent node '{pid}' is not in the tree")

        self[node.id] = node
        if pid in self:
            self[pid].add_child(node.id)
        self[node.id].parent = pid

    def children(self, node_id) -> List[Node]:
        """
        Return the children (Node) list of ``node_id``.

        Empty list is returned if ``node_id`` does not exist.
        """
        return [self[i] for i in self.is_branch(node_id)]

    def create_node(self, *args, parent=None, node_cls=Node, **kwargs):
        """
        Create a new node and add it to this tree.

        If ``id`` is absent, a UUID will be generated automatically.
        """
        if not issubclass(node_cls, Node):
            raise ValueError('node_cls must be a subclass of Node.')

        node = node_cls(*args, **kwargs)
        self.add_node(node, parent)
        return node

    def depth(self, node=None) -> int:
        """
        Get the maximum level of this tree or the level of the given node

        .. note ::

            The parameter is node instance rather than node_identifier.

        :param ~ttree.Node node:
        :return: Depth level
        """
        result = 0
        if node is None:
            # Get maximum level of this tree
            for leaf in self.leaves():
                level = self.level(leaf.id)
                result = level if level >= result else result
        else:
            # Get level of the given node
            node_id = node.id if isinstance(node, Node) else node

            if node_id not in self:
                raise NodeNotFound(f"Node '{node_id}' is not in the tree")

            result = self.level(node_id)
        return result

    def expand_tree(self, node_id=None,
                    mode: Union[TraversalMode, str] = TraversalMode.DEPTH,
                    filtering: Callable[[Node], bool] = None,
                    key=None, reverse: bool = False):
        """
        Traverse the tree nodes with different modes.

        ``node_id`` refers to the expanding point to start; ``mode`` refers
        to the search mode (Tree.DEPTH, Tree.WIDTH). ``filter`` refers to the
        function of one variable to act on the :class:`Node` object.
        In this manner, the traversing will not continue to following children
        of node whose condition does not pass the filter. ``key``, ``reverse``
        are present to sort :class:Node objects at the same level.

        Python generator. Loosly based on an algorithm from
        'Essential LISP' by John R. Anderson, Albert T. Corbett, and
        Brian J. Reiser, page 239-241

        UPDATE: the @filtering function is performed on Node object during
        traversing. In this manner, the traversing will not continue to
        following children of node whose condition does not pass the filter.

        UPDATE: the @key and @reverse are present to sort nodes at each
        level.
        """
        node_id = self.root if node_id is None else node_id

        if node_id not in self:
            raise NodeNotFound(f"Node '{node_id}' is not in the tree")

        if filtering is not None and not callable(filtering):
            raise TypeError('Filtering must be callable.')

        mode = mode if isinstance(mode, TraversalMode) else TraversalMode(mode)

        if filtering is not None and not filtering(self[node_id]):
            return

        yield node_id
        queue = (self[i] for i in self[node_id].children)

        if filtering is not None:
            queue = filter(filtering, queue)

        if mode in (TraversalMode.DEPTH, TraversalMode.WIDTH):
            queue = sorted(queue, key=key, reverse=reverse)
            while queue:
                yield queue[0].id
                expansion = sorted(
                    filter(filtering, (self[i] for i in queue[0].children)),
                    key=key, reverse=reverse
                )

                if mode is TraversalMode.DEPTH:
                    queue = expansion + queue[1:]  # depth-first
                elif mode is TraversalMode.WIDTH:
                    queue = queue[1:] + expansion  # width-first

        elif mode is TraversalMode.ZIGZAG:
            # Suggested by Ilya Kuprik (ilya-spy@ynadex.ru).
            stack_fw = []
            queue = list(queue)
            queue.reverse()
            stack = stack_bw = queue
            direction = False
            while stack:
                expansion = filter(filtering,
                                   (self[i] for i in stack[0].children))
                yield stack.pop(0).id
                expansion = list(expansion)
                if direction:
                    expansion.reverse()
                    stack_bw = expansion + stack_bw
                else:
                    stack_fw = expansion + stack_fw
                if not stack:
                    direction = not direction
                    stack = stack_fw if direction else stack_bw

    def is_branch(self, node_id):
        """
        Get the children (only sons) list of the node with ID == node_id.

        Empty list is returned if node_id does not exist
        """
        if node_id is None:
            raise ValueError("First parameter can't be None")

        return self[node_id].children

    def leaves(self, node_id=None):
        """Get leaves from given node."""
        if node_id is None:
            return [n for n in self.values() if n.is_leaf]

        return [self[n] for n in self.expand_tree(node_id) if self[n].is_leaf]

    def level(self, node_id, filtering: Callable[[Node], bool] = None):
        """
        Get the node level in this tree.
        The level is an integer starting with '0' at the root.
        In other words, the root lives at level '0';

        Update: @filtering params is added to calculate level passing
        exclusive nodes.
        """
        return len([n for n in self.rsearch(node_id, filtering)]) - 1

    def link_past_node(self, node_id):
        """
        Remove a node and link its children to its parent.

        Root is not allowed.

        For example, if we have a -> b -> c and delete node b, we are left
        with a -> c
        """
        if self.root == node_id:
            raise LinkPastRootNode('Cannot link past the root node, '
                                   'delete it with remove_node()')

        # Get the parent of the node we are linking past
        parent = self[self[node_id].parent]

        # Set the children of the node to the parent
        for child in self[node_id].children:
            self[child].parent = parent.id

        # Link the children to the parent
        parent.children += self[node_id].children
        # Delete the node
        parent.remove_child(node_id)
        del self[node_id]

    def move_node(self, source, destination):
        """
        Move node (source) from its parent to another parent (destination).
        """
        if source not in self or destination not in self:
            raise NodeNotFound

        if self.is_ancestor(source, destination):
            raise LoopError

        parent = self[source].parent
        self[parent].remove_child(source)
        self[destination].add_child(source)
        self[source].parent = destination

    def is_ancestor(self, ancestor, grandchild) -> bool:
        parent = self[grandchild].parent
        child = grandchild

        while parent is not None:
            if parent == ancestor:
                return True

            child = self[child].parent
            parent = self[child].parent

        return False

    def parent(self, node_id) -> Optional[Node]:
        """
        Obtain specific node's parent (Node instance).

        Return None if the parent is None or does not exist in the tree.
        """
        pid = self[node_id].parent
        if pid is None or pid not in self:
            return None

        return self[pid]

    def paste(self, node_id, new_tree: 'Tree', deepcopy: bool = False):
        """
        Paste a new tree to an existing tree, with ``node_id`` becoming the
        parent of the root of this new tree.

        Update: add @deepcopy of pasted tree.
        """
        if not isinstance(new_tree, Tree):
            raise TypeError('Instance of Tree is required as '
                            '"new_tree" parameter.')

        if node_id is None:
            raise ValueError('First parameter can not be None')

        if node_id not in self:
            raise NodeNotFound(f"Node '{node_id}' is not in the tree")

        set_joint = set(new_tree) & set(self)  # joint keys
        if set_joint:
            # TODO: a deprecated routine is needed to avoid exception
            raise ValueError(f'Duplicated nodes {list(set_joint)} exists.')

        self.__merge_tree(new_tree, deepcopy)

        self[node_id].add_child(new_tree.root)
        self[new_tree.root].parent = node_id

    def remove_node(self, node_id) -> int:
        """
        Remove a node indicated by 'id'; all the successors are
        removed as well.

        Return the number of removed nodes.
        """
        if node_id is None:
            return 0

        parent = self[node_id].parent

        removed = [n for n in self.expand_tree(node_id)]
        for id_ in removed:
            del self[id_]

        # Update its parent info
        if parent:
            self[parent].remove_child(node_id)

        return len(removed)

    def remove_subtree(self, node_id) -> 'Tree':
        """
        Return a subtree deleted from this tree. If node_id is None, an
        empty tree is returned.
        For the original tree, this method is similar to
        `remove_node(self,node_id)`, because given node and its children
        are removed from the original tree in both methods.
        For the returned value and performance, these two methods are
        different:

            `remove_node` returns the number of deleted nodes;
            `remove_subtree` returns a subtree of deleted nodes;

        You are always suggested to use `remove_node` if your only to
        delete nodes from a tree, as the other one need memory
        allocation to store the new tree.
        """
        subtree = Tree()
        if node_id is None:
            return subtree

        subtree.root = node_id
        parent = self[node_id].parent
        self[node_id].parent = None  # reset root parent for the new tree

        removed = [n for n in self.expand_tree(node_id)]
        for id_ in removed:
            subtree[id_] = self.pop(id_)

        # Update its parent info
        self[parent].remove_child(node_id)
        return subtree

    def rsearch(self, node_id, filtering: Callable[[Node], bool] = None):
        """
        Search the tree from ``node_id`` to the root along links reservedly.

        Parameter ``filter`` refers to the function of one variable to act
        on the :class:`Node` object.
        """
        if node_id is None:
            return

        if node_id not in self:
            raise NodeNotFound(f"Node '{node_id}' is not in the tree")

        if filtering is not None and not callable(filtering):
            raise TypeError('Filtering must be a callable.')

        current = node_id
        while current is not None:
            if filtering is None or filtering(self[current]):
                yield current

            # subtree() hasn't update the parent
            current = self[current].parent if self.root != current else None

    def save2file(self, filename, node_id=None, id_hidden=True,
                  filtering=None, key=None, reverse=False,
                  ascii_mode=ASCIIMode.ex, data_property=None):
        """
        Save the tree into file for offline analysis.

        :param filename: Export file name
        :param node_id: Traversal root node ID
        :param id_hidden: Is ID hidden?
        :param filtering: Filtering callable
        :param key: Sorting key callable
        :param reverse: Reverse mode?
        :param ascii_mode: ASCII mode
        :param data_property: Data property name
        """
        with open(filename, 'ab') as fp:
            ttree.utils.print_tree(
                self, node_id, id_hidden, filtering, key, reverse,
                ascii_mode, data_property, func=lambda n: fp.write(n + b'\n')
            )

    def print(self, node_id=None, id_hidden=True, filtering=None,
              key=None, reverse=False, ascii_mode=ASCIIMode.ex,
              data_property=None):
        """
        Print the tree structure in hierarchy style.

        You have three ways to output your tree data, i.e., stdout with
        ``print()``, plain text file with ``save2file()``, and json string
        with ``to_json()``. The former two use the same backend to generate
        a string of tree structure in a text graph.

        :param node_id: refers to the expanding point to start
        :param id_hidden: refers to hiding the node ID when printing
        :param filtering: refers to the function of one variable to act on the
            :class:`Node` object. In this manner, the traversing will not
            continue to following children of node whose condition does
            not pass the filter.
        :param key: are present to sort :class:`Node` object in the same level
        :param reverse: reverse mode
        :param ascii_mode: ASCII mode
        :param data_property: refers to the property on the node data object
            to be printed.
        """
        try:
            ttree.utils.print_tree(
                self, node_id, id_hidden, filtering,
                key, reverse, ascii_mode, data_property,
                func=print
            )
        except NodeNotFound:
            print('Tree is empty')

    def siblings(self, node_id) -> List[Node]:
        """
        Return the siblings of given ``node_id``.

        If ``node_id`` is root or there are no siblings, an empty list
        is returned.
        """
        siblings = []

        if node_id != self.root:
            pid = self[node_id].parent
            siblings = [self[c] for c in self[pid].children if c != node_id]

        return siblings

    def size(self, level: int = None) -> int:
        """
        Get the number of nodes of the whole tree if @level is not
        given. Otherwise, the total number of nodes at specific level
        is returned.

        @param level The level number in the tree. It must be between
        [0, tree.depth].

        Otherwise, InvalidLevelNumber exception will be raised.
        """
        if level is None:
            return len(self)

        if not isinstance(level, int):
            raise TypeError(f"Level should be an integer instead "
                            f"of '{type(level)}'")

        return len(
            [node for node in self.values()
             if self.level(node.id) == level]
        )

    def subtree(self, node_id) -> 'Tree':
        """
        Return a shallow COPY of subtree with node_id being the new root.
        If node_id is None, return an empty tree.
        If you are looking for a deepcopy, please create a new tree
        with this shallow copy,

        e.g.
            new_tree = Tree(t.subtree(t.root), deep=True)

        This line creates a deep copy of the entire tree.
        """
        result = self.__class__()
        if node_id is None:
            return result

        if node_id not in self:
            raise NodeNotFound(f"Node '{node_id}' is not in the tree")

        result.root = node_id
        for subtree_node in self.expand_tree(node_id):
            result[self[subtree_node].id] = self[subtree_node]

        return result

    def to_dict(self, node_id=None, key=None, sort=True, reverse=False,
                with_data=False) -> MutableMapping:
        """transform self into a dict"""

        node_id = self.root if node_id is None else node_id
        node_tag = self[node_id].tag
        result = {node_tag: {'children': []}}
        if with_data:
            result[node_tag]['data'] = self[node_id].data

        if self[node_id].expanded:
            queue = (self[i] for i in self[node_id].children)
            if sort:
                sort_options = {'reverse': reverse}
                if key is not None:
                    sort_options['key'] = key
                queue = sorted(queue, **sort_options)

            result[node_tag]['children'] = [
                self.to_dict(n.id, with_data=with_data,
                             sort=sort, reverse=reverse)
                for n in queue
            ]

            if not result[node_tag]['children']:
                result = (
                    self[node_id].tag
                    if not with_data
                    else {node_tag: {'data': self[node_id].data}}
                )

        return result

    def to_json(self, with_data=False, sort=True, reverse=False) -> str:
        """Return the json string corresponding to self"""
        return json.dumps(
            self.to_dict(with_data=with_data, sort=sort, reverse=reverse)
        )


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
#
#   for use with Python 3

#	text_buffer
#
#	This program is free software; you can redistribute it and/or modify
#	it under the terms of the GNU General Public License as published by
#	the Free Software Foundation; either version 2 of the License, or
#	(at your option) any later version.
#
#	This program is distributed in the hope that it will be useful,
#	but WITHOUT ANY WARRANTY; without even the implied warranty of
#	MERCHANTABILITY or FITNimport sys, getoptESS FOR A PARTICULAR PURPOSE.  See the
#	GNU General Public License for more details.
#
#	You should have received a copy of the GNU General Public License
#	along with this program; if not, write to the Free Software
#	Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
#	MA 02110-1301, USA.

# Standard library imports
from datetime import datetime
from shutil import copyfile
from sys import exit as sys_exit
import os

# Third party imports
# None

# Local application imports
from utility import pr,make_time_text,send_by_ftp
from buffer_log import class_buffer_log

class class_text_buffer(object):
	# Rotating Buffer Class
	# Initiate with just the size required Parameter
	# Get data with just a position in buffer Parameter
	def __init__(self,headings,config):
		#initialization
		self.__config = config
		print(" Buffer Init for : ",self.__config.prog_name," with a size of : ",self.__config.text_buffer_length, " and  width of : ", len(headings) + 1, " including time stamp")
		if not os.path.exists('log'):
		    os.makedirs('log')

		self.__source_ref = 0 # a number used to control prevention repeat messages
		self.__width = len(headings) + 1
		self.line_values = ["not set"]*(len(headings))
		self.__dta = [ [ None for di in range(self.__width+1) ] for dj in range(self.__config.text_buffer_length+1) ]
		self.__size = 0
		self.__posn = self.__config.text_buffer_length-1

#**********************************************
		#self.__headings = ["Time"]
		#for hdg_ind in range(0,self.__width-1):
		#	#print(hdg_ind,headings[hdg_ind])
		#	self.__headings.append(headings[hdg_ind])
		#print(self.__headings)
#**************************************************
		self.__headings = headings

		self.__pr_values = ["text"] * self.__width

		self.__html_filename = config.prog_name + "_log.html"
		self.__html_filename_save_as = config.prog_path + self.__html_filename
		self.__www_filename = config.local_dir_www +  self.__html_filename
		self.__ftp_creds = config.ftp_creds_filename
		self.__send_html_count = 0
		if self.__config.log_buffer_flag:
			self.__send_log_count = 0
			self.__log = class_buffer_log(config)

	def size(self):
		return self.__config.text_buffer_length

	def update_buffer(self,values,appnd,ref):
		#append a line of info at the current position plus 1
		# print("Update Buffer appnd and ref are : ",appnd,ref)
		if appnd + (self.__source_ref != ref):
			#we adding message and incrementing posn
			if self.__size < self.__config.text_buffer_length-1 :
				self.__size += 1
			if self.__posn == self.__config.text_buffer_length-1:
				# last insert was at the end so go back to beginning@@
				self.__posn = 0
			else:
				# first increment insert position by one
				self.__posn += 1
				# then insert a line full of values
			self.__source_ref = ref
		else:
			self.__source_ref = ref
		if len(values) > self.__width :
			print("Width Error for :",self.__config.prog_name, len(values) , self.__width, values)
			sys.exit()
		for i in range(0,len(values)):
			self.__dta[self.__posn][i] = values[i]

		#print("Buffer updated and log buffer flag is : ",self.__config.log_buffer_flag)
		if self.__config.log_buffer_flag:
			self.__log.log_to_file(self.__headings,values)

			#self.__log.copy_log_to_www(False)
			#send log file to website configevery ten scans

				# *************************
			#if self.__send_log_count > 10:

				#print("Sending log file by FTP")

			#	self.__log.send_log_by_ftp(False,self.__config.log_directory,self.__config.ftp_timeout)
			#	self.__send_log_count = 0
			#else:
				#print("not FTP This Time count is : ",self.__send_log_count)
			#	self.__send_log_count += 1

	def get_line_dta(self, key):
		#return stored element from position relative to current insert position in buffer
		line_dta = [" - "]*self.__width
		if (self.__posn-key) > -1:
			# need to take values from arlogea before current insert position
			for i in range(self.__width):
				line_dta[i] = self.__dta[self.__posn-key][i]
			return(line_dta)
		else:
			# need to take values from after current insert position
			for i in range(self.__width):
				#following two lines used too debug the calc to get the lower part of status file
				#print("That Calc key,self.__size,self.__config.text_buffer_length, self.__posn-key,key sum",
				 #  key,self.__size,self.__config.text_buffer_length, self.__posn-key,(self.__posn-key),self.(self.__config.text_buffer_length + (self.__posn-key))
				line_dta[i] = self.__dta[self.__config.text_buffer_length + (self.__posn-key)][i]
			return(line_dta)

	def get_dta(self):
		# get all the data inserted so far, or the whole buffer
		all_data = [ [ None for di in range(self.__width+1) ] for dj in range(self.__config.text_buffer_length+1) ]
		for ind in range(0,self.__size):
			line_dta = self.get_line_dta(ind)
			# Following line for debug data from Buffer
			# print("get_dta >>>",ind,line_dta)
			for i in range(len(line_dta)):
				all_data[ind][i] = line_dta[i]
		return(all_data)

	def just_log(self,appnd,ref,log_time,refresh_interval):
		self.__log.log_to_file(self.__headings,self.line_values)

	def pr(self,appnd,ref,log_time,refresh_interval):
		here = "buffer.pr for " + self.__config.prog_name
		make_values = [" -- "]*self.__width
		prtime = datetime.now()
		for_screen = log_time.strftime('%d/%m/%Y %H:%M:%S')
		# following alternative will show more resolution for fractions of a second
		# for_screen = log_time.strftime('%d/%m/%Y %H:%M:%S.%f')
		make_values[0] = for_screen
		file_start = """
<meta http-equiv="refresh" content="""
		file_start = file_start + str(refresh_interval)
		file_start = file_start + """ />

Rotating Buffer Display
"""
		tbl_start = """

"""
		tbl_start_line = """
"""
		tbl_end_line = """
"""
		tbl_start_col = """"""
		tbl_end_col= """	"""
		tbl_end = """

"""
		file_end = """

"""
		try:
			for i in range(0,self.__width -1):
				make_values[i+1] = str(self.line_values[i])
				for_screen = for_screen + " " + str(self.line_values[i])
		except:
			print("Error in make values in ...buffer.pr for : ",self.__config.prog_name)
			print("i,values,len(self.line_value>s),self.__width",i,self.line_values,len(self.line_values),self.__width)
			sys_exit()

		# print to screen and to status log and update html file

		if appnd:
			print("    appending : " + self.__config.prog_name + " : " + for_screen)
		else:
			print("not appending : " + self.__config.prog_name + " : " + for_screen)

		self.update_buffer(make_values,appnd,ref)
		with open(self.__html_filename,'w') as htmlfile:
			htmlfile.write(file_start)
			htmlfile.write("
" + self.__html_filename + " : " + make_time_text(datetime.now())  + "
\n
")
			htmlfile.write(tbl_start + tbl_start_line)
			for ind in range(0,len(self.__headings)):
				htmlfile.write(tbl_start_col + self.__headings[ind] + tbl_end_col)
			htmlfile.write(tbl_end_line)
			buffer_dta = self.get_dta()
			for ind in range(self.__size):
				htmlfile.write(tbl_start_line)
				for i in range(self.__width):
					htmlfile.write(tbl_start_col + str(buffer_dta[ind][i]) + tbl_end_col)
				htmlfile.write(tbl_end_line)
			htmlfile.write(tbl_end)
			htmlfile.write(file_end)
		copyfile(self.__html_filename, self.__www_filename)

		# To debug FTP change end of following line to " = True"
		if self.__send_html_count >= 3:
			FTP_dbug_flag = False
			ftp_result = send_by_ftp(FTP_dbug_flag,self.__ftp_creds, self.__html_filename_save_as, self.__html_filename,"",self.__config.ftp_timeout)
			for pres_ind in range(0,len(ftp_result)):
				pr(FTP_dbug_flag,here, str(pres_ind) + " : ", ftp_result[pres_ind])
			self.__send_html_count = 0
		else:
			self.__send_html_count += 1
		return


import datetime, json, logging, os, codecs, re
import unicodecsv as csv
import textmining, mediacloud, stopwords
from ConfigParser import SafeConfigParser

ROWS_PER_QUERY = 500000

# Load config data
parser = SafeConfigParser()
parser.read('config.txt')
MY_API_KEY = parser.get('API','MY_API_KEY')
mc = mediacloud.api.AdminMediaCloud(MY_API_KEY) #AdminMediaCloud, rather than MediaCloud

logging.basicConfig(level=logging.DEBUG)
logging.info("-----------------------------------------------------------------")
logging.info("Starting QB data gathering")

# build stopwords
my_stopwords = [word.lower() for word in stopwords.getStopWords()]
qb_table = csv.reader(codecs.open('qb-table.csv', 'r', 'utf-8'))
qb_table.next()
team_stopwords = []
qb_stopwords = []
for row in qb_table:
    [ team_stopwords.append(word.lower()) for word in row[0].split() ]
    [ qb_stopwords.append(word.lower()) for word in row[1].split() ]
logging.debug(" Added qb names to stopwords: %s" % qb_stopwords)
logging.debug(" Added team names to stopwords: %s" % team_stopwords)
my_stopwords = my_stopwords + qb_stopwords + team_stopwords

# load media sources
m = codecs.open('sources.csv','r','utf-8')
media_reader = csv.reader(m)
media = [x[1] for x in media_reader][1:]
media_id_str = " ".join(media)
logging.info("Searching in %d media" % len(media))
logging.debug("media ids = %s" % media_id_str)

def fetch_corpus_from_mc(team,qb): #MC query, returns list of words
    '''
    Query MC for coverage of the QB specified, return a large string corpus
    '''
    more = True
    start = 0
    while more:
        logging.debug('    starting at %d' % start)
        sentences = mc.sentenceList(solr_query=str('"'+qb+'"'),
            solr_filter=[mc.publish_date_query(datetime.date(2015,9,9), datetime.date(2016,1,4)),
                         '+media_id:('+media_id_str+')'], rows = ROWS_PER_QUERY, start=0)
        more = len(sentences['response']['docs'])==ROWS_PER_QUERY
        start = start + ROWS_PER_QUERY
    logging.info('    found %d sentences',len(sentences['response']['docs']))
    response = sentences['response']
    docs = response['docs']
    logging.info('  done')
    return " ".join([d['sentence'] for d in docs])

def load_qb_corpus(team, qb):
    '''
    Grab the corpus for a QB from local data, or from MC if we don't have it yet
    '''
    corpora_dir = os.path.join('data','corpora')
    qb_corpora_path = os.path.join(corpora_dir,qb+'.txt')
    if not os.path.exists(corpora_dir):
        os.makedirs(corpora_dir)
    if os.path.exists(qb_corpora_path):
        f = codecs.open(qb_corpora_path, 'r', encoding='utf-8')
        corpus = f.read()
    else:
        corpus = fetch_corpus_from_mc(team,qb)
        f = codecs.open(qb_corpora_path, 'w', encoding='utf-8')
        f.write(corpus)
    return corpus

def tokenize_and_remove_stopwords(document):
    '''
    Callback to help the TDM creation via the textmining package
    '''
    document = document.lower() # do everything in lowercase
    document = re.sub('[^a-z]', ' ', document)  # remove non-alpha-numeric chars
    words = document.strip().split()
    before_word_count = len(words)
    # Remove stopwords
    words = [word for word in words if word not in my_stopwords]
    after_word_count = len(words)
    logging.debug("    %d (removed %d)" % (after_word_count,before_word_count-after_word_count))
    return words

def write_csv(cols, doc_iterator, filename):
    '''
    Write the results to a CSV in a human-usable format
    '''
    words = doc_iterator.next()
    word_freqs = [ row for row in doc_iterator ]
    corpus_word_count = [sum(row) for row in word_freqs]
    output_csv = csv.writer( codecs.open( os.path.join('data',filename), 'wb', 'utf-8') )
    output_csv.writerow(['word']+cols+[c+" pct" for c in cols])
    for idx in range(0,len(words)):
        word_counts = [ r[idx] for r in word_freqs ]
        normalized_word_counts = [ float(r[idx])/float(corpus_word_count[i]) for i,r in enumerate(word_freqs) ]
        output_csv.writerow([words[idx]]+word_counts+normalized_word_counts)

def get_and_write_data():
    '''
    Call this to collect and output the data
    '''
    corpus = {}
    count_corpus = {}
    white_doc = ""
    other_doc = ""
    logging.info('Loading sentences...')
    tdm_names = []
    tdm = textmining.TermDocumentMatrix(tokenize_and_remove_stopwords)
    race_tdm_names = []
    race_tdm = textmining.TermDocumentMatrix(tokenize_and_remove_stopwords)
    qb_table = csv.reader(codecs.open('qb-table.csv', 'r', 'utf-8'))
    qb_table.next()
    for row in qb_table:
        team = row[0]
        qb = row[1]
        race = row[2]
        logging.info('  %s (%s) - %s' % (qb,team,race))
        file_label = str(qb+' ('+team+')')
        qb_corpus = load_qb_corpus(team,qb)
        tdm.add_doc(qb_corpus)
        tdm_names.append(qb)
        if race == 'white':
            white_doc += qb_corpus + " "
        else:
            other_doc += qb_corpus + " "
    logging.info('done')
    # write the results
    logging.info('Writing results...')
    write_csv(tdm_names, tdm.rows(cutoff=1), 'word_freq_by_quarterback.csv')
    logging.info("Building white corpus")
    race_tdm.add_doc(white_doc)
    logging.info("Building non-white corpus")
    race_tdm.add_doc(other_doc)
    write_csv(['white','other'], race_tdm.rows(cutoff=1), 'word_freq_by_race.csv')
    logging.info('done')

if __name__ == "__main__":
    get_and_write_data()


#executemany() method

import sqlite3

with sqlite3.connect("newdb.db") as connection:
	c = connection.cursor()

	#insert multiple records using a tuple
	cities = [
			('Boston', 'MA', 250201),
			('Houston', 'TX', 800000),
			('Hong Kong', 'HK', 900000),
			('Phoenix', 'AZ', 150000)
			]
	c.executemany('INSERT INTO population VALUES(?, ?, ?)', cities)


import collections
import configparser
import hashlib
import os
import re
import zlib

git_folder = ".wyag" # ".git"

class GitRepository(object):
    """A git repository"""

    worktree = None
    gitdir = None
    conf = None

    def __init__(self, path, force=False):
        self.worktree = path
        self.gitdir = os.path.join(path, git_folder)

        if not (force or os.path.isdir(self.gitdir)):
            raise Exception("Not a Git repository {:s}".format(self.worktree))

        # read configuration file .git/config
        self.conf = configparser.ConfigParser()
        cf = repo_file(self, "config")

        if cf and os.path.exists(cf):
            self.conf.read([cf])
        elif not force:
            raise Exception("Configuration file is missing")

        if not force:
            vers = int(self.conf.get("core", "repositoryformatversion"))
            if vers != 0:
                raise Exception("Unsupported repositoryformatversion {}".format(vers))

def repo_path(repo, *path):
    """Comput path under repo's gitdir"""
    return os.path.join(repo.gitdir, *path)

def repo_file(repo, *path, mkdir=False):
    """Same as repo_path, but create dirname(*path) if absent. For example,
    repo_file(r, refs, remotes, origin, HEAD) will create .git/refs/remotes/origin."""

    if repo_dir(repo, *path[:-1], mkdir=mkdir):
        return repo_path(repo, *path)

def repo_dir(repo, *path, mkdir=False):
    """Same as repo_path, but mkdir *path if absent if mkdir."""

    path = repo_path(repo, *path)

    if os.path.exists(path):
        if os.path.isdir(path):
            return path
        else:
            raise Exception("Not a directory {}".format(path))

    if mkdir:
        os.makedirs(path)
        return path
    else:
        return None

def repo_create(path):
    """Create a new repository at path."""

    repo = GitRepository(path, True)

    # make sure path doesn't exist or is empty

    if os.path.exists(repo.worktree):
        if not os.path.isdir(repo.worktree):
            raise Exception("{} is not a directroy".format(path))
        if os.listdir(repo.worktree):
            raise Exception("{} is not empty".format(path))
    else:
        os.makedirs(repo.worktree)

    assert(repo_dir(repo, "branches", mkdir=True))
    assert(repo_dir(repo, "objects", mkdir=True))
    assert(repo_dir(repo, "refs", "tags", mkdir=True))
    assert(repo_dir(repo, "refs", "head", mkdir=True))

    # .git/description
    with open(repo_file(repo, "description"), "w") as f:
        f.write("Unnamed repository; edit this file 'description' to name the repository.\n")

    # .git/HEAD
    with open(repo_file(repo, "HEAD"), "w") as f:
        f.write("refs: refs/heads/master\n")

    with open(repo_file(repo, "config"), "w") as f:
        config = repo_default_config()
        config.write(f)

    return repo

def repo_default_config():
    ret = configparser.ConfigParser()

    ret.add_section("core")
    ret.set("core", "repositoryformatversion", "0")
    ret.set("core", "filemode", "false")
    ret.set("core", "bare", "false")

    return ret

def repo_find(path=".", required=True):
    path = os.path.realpath(path)

    if os.path.isdir(os.path.join(path, git_folder)):
        return GitRepository(path)

    # search recursive in parent directories
    parent = os.path.realpath(os.path.join(path, ".."))

    if parent == path:
        # re reached root

        if required:
            raise Exception("No git repository")
        else:
            return None

    return repo_find(parent, required)

class GitObject (object):

    repo = None

    def __init__(self, repo, data=None):
        self.repo = repo

        if data != None:
            self.deserialize(data)

    def serialize(self):
        """This function MUST be implemented by subclassses.

        It must read the obkect's contens from self.data, a byte string, and
        do whaterver it takes to conber it into a meaningful representation.
        What exactly that means depends on each subclass."""

        raise Exception("Unimplemented!")

    def deserialize(self, data):
        raise Exception("Unimplemented!")

def object_find(repo, name, fmt=None, follow=True):
    return name

def object_read(repo, sha):
    """Read object object_id from Git repository repo. Return a GitObject
    whose exact type depends on the object."""

    path = repo_file(repo, "objects", sha[:2], sha[2:])

    with open(path, "rb") as f:
        raw = zlib.decompress(f.read())

        # read object type
        x = raw.find(b' ')
        fmt = raw[0:x]

        # read and validate  object size
        y = raw.find(b'\x00', x)
        size = int(raw[x:y].decode("ascii"))
        if size != len(raw)-y-1:
            raise Exception("Malformed object {}: bad length".format(sha))

        # pick correct constructor
        if fmt==b'commit' : c=GitCommit
        elif fmt==b'tree' : c=GitTree
        elif fmt==b'tag' : c=GitTag
        elif fmt==b'blob' : c=GitBlob
        else:
            raise Exception("Unknown type {} for object {}".format(fmt.decode("ascii"), sha))

        return c(repo, raw[y+1:])

def object_write(obj, actually_write=True):
    # serialize object data
    data = obj.serialize()

    # add header
    result = obj.fmt + b' ' + str(len(data)).encode() + b'\x00' + data

    # get hash
    sha = hashlib.sha1(result).hexdigest()

    if actually_write:
        # compute path
        path = repo_file(obj.repo, "objects", sha[:2], sha[2:], mkdir=actually_write)

        with open(path, 'wb') as f:
            # compress and write
            f.write(zlib.compress(result))

    return sha

def object_hash(f, fmt, repo=None):
    data = f.read()

    # Choose constructor depnedo on object type foound in header
    if fmt == b'commit' : obj=GitCommit(repo, data)
    elif fmt == b'tree' : obj=GitTree(repo, data)
    elif fmt == b'tag' : obj=GitTag(repo, data)
    elif fmt == b'blob' : obj=GitBlob(repo, data)
    else:
        raise Exception("Unknown type {}".format(fmt))

    return object_write(obj, repo)

class GitBlob(GitObject):
    fmt=b'blob'

    def serialize(self):
        return self.blobdata

    def deserialize(self, data):
        self.blobdata = data

# Key-Value List with Message
def kvlm_parse(raw, start=0, dct=None):
    if not dct:
        dct = collections.OrderedDict()
        # you cannot declare the argument as dct=OrderedDict() or all
        # call to the function will endlessly grom the same dict

    # we search for the next space and the next newline
    spc = raw.find(b' ', start)
    nl = raw.find(b'\n', start)

    # id space appears before the newline, we have a keyword

    # base case
    # ============
    # If newline appears first(or there is no space at all, in which
    # case find returns -1), we assume a blank line. A blank line
    # means the remainder of the data is the message.
    if (spc < 0) or (nl < spc):
        assert(nl == start)
        dct[b''] = raw[start+1:]
        return dct

    # Recusive case
    # ================
    # we read a key-value pair and recurse for the next.
    key = raw[start:spc]

    # Find the end of the value. Continuatuon lines begin with a
    # space, so loop until we find a \n not followed by a space.
    end = start
    while True:
        end = raw.find(b'\n', end + 1)
        if raw[end + 1] != ord(' '):
            break

    # Grab the value
    # Also, drop the leading space on continuation lines
    value = raw[spc + 1:end].replace(b'\n ', b'\n')

    # do not overwrite existing data contents
    if key in dct:
        if type(dct[key]) == list:
            dct[key].append(value)
        else:
            dct[key] = [dct[key], value]
    else:
        dct[key] = value

    return kvlm_parse(raw, start=end+1, dct=dct)

def kvlm_serialize(kvlm):
    ret = b''

    # output fields
    for k in kvlm.keys():
        # skip message itself
        if k==b'':
            continue
        val = kvlm[k]
        # Normalize to a list
        if type(val) != list:
            val = [val]

        for v in val:
            ret += k + b' ' + v.replace(b'\n', b'\n ') + b'\n'

    # append message
    ret += b'\n' + kvlm[b'']

    return ret

class GitCommit(GitObject):
    fmt = b'commit'

    def deserialize(self, data):
        self.kvlm = kvlm_parse(data)

    def serialize(self):
        return kvlm_serialize(self.kvlm)

class GitTreeLeaf(object):
    def __init__(self, mode, path, sha):
        self.mode = mode
        self.path = path
        self.sha = sha

def tree_parse_one(raw, start=0):
    # find the space terminator of the mode
    x = raw.find(b' ', start)
    assert(x-start == 5 or x-start == 6)

    # read the mode
    mode = raw[start:x]

    # fin the NULL terminator of the path
    y = raw.find(b'\x00', x)

    # read the path
    path = raw[x+1:y]

    # read SHA as binary and convert to hex string
    sha = hex(
        int.from_bytes(
            raw[y+1:y+21], "big"))[2:] # hex add 0x in front, we need to remove that

    return y+21, GitTreeLeaf(mode, path, sha)

def tree_parse(raw):
    pos = 0
    max = len(raw)
    ret = list()
    while pos < max:
        pos, data = tree_parse_one(raw, pos)
        ret.append(data)

    return ret

def tree_serialize(obj):
    ret = b''
    for i in obj.items:
        ret += i.mode
        ret += b' '
        ret += i.path
        ret += b'\x00'
        sha = int(i.sha, 16)
        ret += sha.to_bytes(20, byteorder="big")
    return ret

class GitTree(GitObject):
    fmt = b'tree'

    def deserialize(self, data):
        self.items = tree_parse(data)

    def serialize(self):
        return tree_serialize(self)