from pyspark import SparkConf, SparkContext
from pyspark.sql import SparkSession
from pyspark.sql import SQLContext
from pyspark.sql.functions import *
from pyspark.sql.types import *
import time
import datetime
conf = SparkConf(). \
setAppName("Streaming Data"). \
setMaster("yarn-client")
sc = SparkContext(conf=conf)

sqlcontext = SQLContext(sc)

cluster_name = "gs://cluster_name" # cluster in which kafka & zookeeper are installed

bucket_name = "gs://your_bucket_name"

spark = SparkSession \
    .builder \
    .appName("user-logs-analysis-streaming") \
    .getOrCreate()

df = spark \
      .readStream \
      .format("kafka") \
      .option("kafka.bootstrap.servers", cluster_name+"-w-1:9092") \
      .option("subscribe", "user_browsing_logs") \
      .option("failOnDataLoss","false") \
      .load() \
      .selectExpr("CAST(value AS STRING) ")

schema = StructType(
    [
        StructField('category', StringType(), True),
        StructField('date_time', TimestampType(), True),
        StructField('type', StringType(), True),
        StructField('pid', IntegerType(), True),
        StructField('state', StringType(), True),
        StructField('sub_cat', StringType(), True),
        StructField('ip_address', StringType(), True)
   ]
)

df_parsed = df.select("value")

df_streaming_visits = df_parsed.withColumn("data", from_json("value",schema)).select(col('data.*'))

def foreach_batch_function(df,epoch_id) :
    print(epoch_id)
    print("--------------------")
    df.show()

    df.coalesce(1).write \
    .format("parquet") \
    .mode("append") \
    .option("checkpointLocation",bucket_name+"/spark-checkpoints") \
    .option("path",bucket_name+"/streaming_data_output") \
    .save()

query = df_streaming_visits.writeStream \
            .trigger(processingTime="10 seconds") \
            .foreachBatch(foreach_batch_function) \
            .outputMode("append") \
            .start()

query.awaitTermination()


#!/usr/bin/env python

from sys import stdin, stdout
import argparse

class CustomFormatter(argparse.RawDescriptionHelpFormatter):
	pass

def parse_arguments():
	parser = argparse.ArgumentParser(description='Colorize \033[1;36mA\033[0m\033[1;31mC\033[0m\033[1;32mG\033[0m\033[1;33mT\033[0m\033[0;37mN\033[0m from STDIN and print to STDOUT.', add_help=True, formatter_class=CustomFormatter)
	return parser.parse_args()

def colorize(base):
	if base.upper() == 'A':
		return '\033[1;36m{}\033[0m'.format(base)
	if base.upper() == 'C':
		return '\033[1;31m{}\033[0m'.format(base)
	if base.upper() == 'G':
		return '\033[1;32m{}\033[0m'.format(base)
	if base.upper() == 'T':
		return '\033[1;33m{}\033[0m'.format(base)
	else:
		return '\033[0;37m{}\033[0m'.format(base)

if __name__ == '__main__':
	args = parse_arguments()
	for line in stdin:
		for x in line:
			stdout.write(colorize(x))


import datetime
import os
import string
import time

timestrings = [
    "%a, %d %b %Y %H:%M:%S %z",
    "%d %b %Y %H:%M:%S %z",
    "%d %b %Y %H:%M:%S",
    "%a, %d %b %Y %H:%M:%S",
    "%d %b %a %H:%M:%S %Y %Z",
    "%d %b %a %H:%M:%S %Y %z",
    "%a %d %b %H:%M:%S %Y %z",
    "%a %b %d %H:%M:%S %Y",
    "%d %b %Y %H:%M:%S",
    "%a %b %d %H:%M:%S %Y",
    "%Y-%m-%d %H:%M:%S",
    "%Y-%m-%dt%H:%M:%S+00:00",
    "%a, %d %b %Y %H:%M:%S +0000",
    "%d %b %Y %H:%M:%S +0000",
   "%d, %b %Y %H:%M:%S +0000"
]

year_formats = [
    "%b %H:%M",
    "%b %H:%M:%S",
    "%a %H:%M %Y",
    "%a %H:%M",
    "%a %H:%M:%S",
    "%Y-%m-%d",
    "%d-%m-%Y",
    "%d-%m",
    "%m-%d",
    "%Y-%m-%d %H:%M:%S",
    "%d-%m-%Y %H:%M:%S",
    "%d-%m %H:%M:%S",
    "%m-%d %H:%M:%S",
    "%Y-%m-%d %H:%M",
    "%d-%m-%Y %H:%M",
    "%d-%m %H:%M",
    "%m-%d %H:%M",
    "%H:%M:%S",
    "%H:%M"
]

allowedchars = string.ascii_letters + string.digits + '_+/$.-'

def day():
    return str(datetime.datetime.today()).split()[0]

def days(path):
    return elapsed(time.time() - fntime(path))

def elapsed(seconds, short=True):
    txt = ""
    nsec = float(seconds)
    year = 365*24*60*60
    week = 7*24*60*60
    nday = 24*60*60
    hour = 60*60
    minute = 60
    years = int(nsec/year)
    nsec -= years*year
    weeks = int(nsec/week)
    nsec -= weeks*week
    nrdays = int(nsec/nday)
    nsec -= nrdays*nday
    hours = int(nsec/hour)
    nsec -= hours*hour
    minutes = int(nsec/minute)
    sec = nsec - minutes*minute
    if years:
        txt += "%sy" % years
    if weeks:
        nrdays += weeks * 7
    if nrdays:
        txt += "%sd" % nrdays
    if years and short and txt:
        return txt
    if hours:
        txt += "%sh" % hours
    if nrdays and short and txt:
        return txt
    if minutes:
        txt += "%sm" % minutes
    if hours and short and txt:
        return txt
    if sec == 0:
        txt += "0s"
    elif sec < 1 or not short:
        txt += "%.3fs" % sec
    else:
        txt += "%ss" % int(sec)
    txt = txt.strip()
    return txt

def fntime(daystr):
    daystr = daystr.replace("_", ":")
    datestr = " ".join(daystr.split(os.sep)[-2:])
    try:
        datestr, rest = datestr.rsplit(".", 1)
    except ValueError:
        rest = ""
    try:
        t = time.mktime(time.strptime(datestr, "%Y-%m-%d %H:%M:%S"))
        if rest:
            t += float("." + rest)
    except ValueError:
        t = 0
    return t

def get_time(daystr):
    for f in year_formats:
        try:
            t = time.mktime(time.strptime(daystr, f))
            return t
        except Exception:
            pass

def now():
    return str(datetime.datetime.now()).split()[0]

def parse_date(daystr):
    if daystr.startswith("-"):
        neg = True
        daystr = daystr[1:]
    else:
        neg = False
    val = 0
    total = 0
    for c in daystr:
        if c not in ["s", "m", "h", "d", "w", "y"]:
            try:
                val = int(c)
            except ValueError:
                pass
            continue
        if c == "y":
            total += val * 3600*24*365
        if c == "w":
            total += val * 3600*24*7
        elif c == "d":
            total += val * 3600*24
        elif c == "h":
            total += val * 3600
        elif c == "m":
            total += val * 60
        else:
            total += val
        val = 0
    if neg:
        return 0 - total
    else:
        return total

def rtime():
    res = str(datetime.datetime.now()).replace(" ", os.sep)
    return res

def today():
    return datetime.datetime.today().timestamp()

def to_day(daystring):
    line = ""
    daystr = str(daystring)
    for word in daystr.split():
        if "-" in word:
            line += word + " "
        elif ":" in word:
            line += word
    if "-" not in line:
        line = day() + " " + line
    try:
        return get_time(line.strip())
    except ValueError:
        pass

def to_time(daystr):
    daystr = daystr.strip()
    if "," in daystr:
        daystr = " ".join(daystr.split(None)[1:7])
    elif "(" in daystr:
        daystr = " ".join(daystr.split(None)[:-1])
    else:
        try:
            d, h = daystr.split("T")
            h = h[:7]
            daystr = " ".join([d, h])
        except (ValueError, IndexError):
            pass
    res = 0
    for tstring in timestrings:
        try:
            res = time.mktime(time.strptime(daystr, tstring))
            break
        except ValueError:
            try:
                res = time.mktime(time.strptime(" ".join(daystr.split()[:-1]), tstring))
            except ValueError:
                pass
        if res:
            break
    return res


from database import *
from SQLoperation import *
from exceptions import *
from condition import *
from antlr4 import *
from .miniSQLParser import miniSQLParser
from functools import reduce
from copy import copy

class Visitor(ParseTreeVisitor):

    def __init__(self):
        ParseTreeVisitor.__init__(self)

        self.relationNames = dict() # alias -> real name
        self.attributeNames = dict() # alias -> Attribute(table,name)
        self.attributeNeeded = dict() # alias -> Attribute(table,name)
        self.aggregate = dict() # name -> operator

        self.allAttr = False
        self.hasAggreg = False

        self.dataManager = DataManager() # will handle relation loading

    def reset(self):
        print_debug("Reseting all attributes of visitor class")
        self.relationNames = dict()
        self.attributeNames = dict()
        self.aggregate = dict()
        self.allAttr = False
        self.hasAggreg = False
        self.dataManager = DataManager()

    def visitMain(self, ctx:miniSQLParser.MainContext):
        print_debug("visitMain")
        resultRelation = self.visit(ctx.sql())
        self.dataManager.discard()
        return resultRelation

    # ________________________ sql rules _______________________________________
    def visitSqlNormal(self, ctx:miniSQLParser.SqlNormalContext):
        # SELECT (DISTINCT)? atts FROM rels (WHERE cond)? (GROUPBY att)? orderby?
        print_debug("visitSqlNormal")

        # 1/ Get the tables in main query
        relations = self.visit(ctx.rels())

        # 2/ Get eventual tables from subqueries
        if ctx.cond() is not None:
            condTree,relList,_ = self.visit(ctx.cond())
            print_debug("Relations from cond:", relList)
            relations.update(relList)

        # 3/ Getting attributes
        attributes = self.visit(ctx.atts())
        print_debug(" Attributes :", attributes)
        print_debug(" Aggregates :", self.aggregate)

        # 4/ Only one relation => using special operators
        if len(relations)==1:
            attributes = self.visit(ctx.atts())

            if not self.aggregate:
                print_debug("Calling readSelectProjectRename")
                rel = relations.pop()
                condTree = None
                if ctx.cond() is not None:
                    condTree,_,_ = self.visit(ctx.cond())
                return readSelectProjectRename(rel[0], rel[1], attributes, self.allAttr, Or(condTree))

            elif ctx.GROUPBY() is not None:
                print_debug("Calling readSelectRenameGroupByProject")
                rel = relations.pop()
                condTree = None
                if ctx.cond() is not None:
                    condTree,_,_ = self.visit(ctx.cond())
                grpAttr = self.visit(ctx.att())
                return readSelectRenameGroupByProject(rel[0], rel[1], attributes, self.allAttr, Or(condTree), grpAttr, self.aggregate)

        # 5/ Load all the tables
        print_debug(" Relations 0:" + str(self.relationNames))
        temp = set()
        for (fileName,tableName) in relations :
            self.dataManager.load(fileName)
            self.relationNames[tableName]=fileName
            if fileName!=tableName :
                self.dataManager.rename_table(fileName,tableName)
            temp.add(tableName) # get rid of fileName : not useful from now on
        relations = temp
        print_debug(" Relations :" + str(self.relationNames))

        # 6/ Completing attributeNeeded, if is a 'SELECT *'
        if self.allAttr :
            for tableName,fileName in self.visit(ctx.rels()) :
                for attribute in self.dataManager(tableName).get_keys() :
                    self.attributeNeeded[attribute.fullName] = attribute
        print_debug(" AttributeNeeded :", self.attributeNeeded)

        # 7/ Perform a join on the tables, only on usefull attributes
        def simplif(x):
            return project( x, [attr for attr in self.attributeNeeded.values() if attr.table == x.name ] )
        tables = [simplif(x) for x in self.dataManager.get_tables(relations)]
        resultRelation = reduce(lambda x,y : join(x,y), tables)

        # 8/ 2nd pass over conditions to get condition tree
        if ctx.cond() is not None:
            condTree,_,notIn = self.visit(ctx.cond())
            if notIn:
                r1,r2 = Table.duplicate(resultRelation)
                resultNormal = select(r1, Or(condTree), True)
                #print_debug("-------------------------------Le normal : ---------------------------------------")
                #print_debug(resultNormal)

                resultWithIn = select(r2, Or(condTree), False)
                #print_debug("-------------------------------Le withIn : ---------------------------------------")
                #print_debug(resultWithIn)

                resultRelation = minus(resultNormal,resultWithIn)
                #print_debug("-------------------------------Le final : ---------------------------------------")
                #print_debug(resultRelation)
            else:
                resultRelation = select(resultRelation, Or(condTree))
            print_debug(" Conditions : " + str(condTree))

        # 9/ Sorting of output
        if ctx.orderby() is not None :
            orderkeys,desc = self.visit(ctx.orderby())
            print_debug("Order keys : ", orderkeys)
            resultRelation = orderBy(resultRelation, orderkeys, desc=desc)

        # 10/ Group By or aggregate result
        if ctx.GROUPBY() is not None:
            print_debug("Group By")
            grpAttr = self.visit(ctx.att())
            resultRelation = groupBy(resultRelation, grpAttr, self.aggregate)
        elif self.hasAggreg:
            resultRelation = aggregate(resultRelation, self.aggregate)

        # 11/ Perform final projection if not a 'SELECT *'
        if not self.allAttr:
            resultRelation = project(resultRelation, attributes)

        # 12/ Delete duplicates
        if ctx.DISTINCT() is not None :
            print_debug("Select Distinct")
            resultRelation = select_distinct(resultRelation)

        return resultRelation

    def visitSqlMinus(self, ctx:miniSQLParser.SqlMinusContext):
        # LPAR sql RPAR MINUS LPAR sql RPAR
        print_debug("visitSqlMinus")
        rel1 = self.visit(ctx.sql(0))
        self.reset()
        rel2 = self.visit(ctx.sql(1))
        return minus(rel1,rel2)

    def visitSqlUnion(self, ctx:miniSQLParser.SqlUnionContext):
        # LPAR sql RPAR UNION LPAR sql RPAR
        print_debug("visitSqlUnion")
        rel1 = self.visit(ctx.sql(0))
        self.reset()
        rel2 = self.visit(ctx.sql(1))
        return union(rel1,rel2)

    # _________________________ subsql rules ___________________________________
    def visitSubSql(self, ctx, attr):
        """
        Called when visiting a sub query
        """
        print_debug("visitSubSql")
        clue = ctx.getChild(3).getText()
        if clue == 'MINUS':
            return self.visitSubSqlMinus(ctx,attr)
        elif clue == 'UNION':
            return self.visitSubSqlUnion(ctx,attr)
        else:
            return self.visitSubSqlNormal(ctx,attr)

    def visitSubSqlNormal(self, ctx, attr):
        print_debug("visitSubSqlNormal")
        attributes = self.visit(ctx.atts())
        assert(len(attributes)==1) # There should be only one here
        subAttr = attributes[0]
        relations = self.visit(ctx.rels())
        condTree, rels, _ = self.visit(ctx.cond()) # condTree a list of list : CDF form
        return [Or(condTree),Condition(attr,Op.EQ,subAttr)], (relations | rels)

    def visitSubSqlMinus(self, ctx, attr): # TODO : false version
        print_debug("visitSubSqlUnion")
        condTree1, rels1 = self.visitSubSql(ctx.sql(0), attr)
        condTree2, rels2 = self.visitSubSql(ctx.sql(1), attr)
        return [Or([And(condTree1),And(condTree2)])], (rels1 | rels2)

    def visitSubSqlUnion(self, ctx, attr):
        print_debug("visitSubSqlUnion")
        condTree1, rels1 = self.visitSubSql(ctx.sql(0), attr)
        condTree2, rels2 = self.visitSubSql(ctx.sql(1), attr)
        return [Or([And(condTree1),And(condTree2)])], (rels1 | rels2)

    # _________________________ Order By rules _____________________________________

    def visitOrderBy(self, ctx:miniSQLParser.OrderByContext):
        # ORDERBY atts (DESC|ASC)?
        orderkeys = self.visit(ctx.atts())
        desc = ctx.DESC() is not None
        return orderkeys,desc

    # _________________________ atts rules _____________________________________
    def visitAttributeDeclAll(self, ctx:miniSQLParser.AttributeDeclAllContext):
        # SELECT *
        print_debug("visitAttributeDeclAll")
        self.allAttr = True
        return []

    def visitAttributeDeclList(self, ctx:miniSQLParser.AttributeDeclListContext):
        print_debug("visitAttributeDeclList")
        queueOfAttrList = self.visit(ctx.atts())
        firstAttrElem, aggr =  self.visit(ctx.attd())
        self.aggregate[firstAttrElem] = aggr
        return [firstAttrElem]+queueOfAttrList

    def visitAttributeDeclSimple(self, ctx:miniSQLParser.AttributeDeclSimpleContext):
        print_debug("visitAttributeDeclSimple")
        attr, aggr = self.visit(ctx.attd())
        self.aggregate[attr] = aggr
        return [attr]

    # ___________________ attd rules ___________________________________________
    def visitAttributeSimple(self, ctx:miniSQLParser.AttributeSimpleContext):
        print_debug("visitAttributeSimple")
        attr = self.visit(ctx.att())
        self.attributeNames[attr.fullName] = attr
        self.attributeNeeded[attr.fullName] = attr
        return attr, Aggregation.NONE

    def visitAttributeAs(self, ctx:miniSQLParser.AttributeAsContext):
        print_debug("visitAttributeAs")
        attr = self.visit(ctx.att())
        alias = ctx.ID().getText()
        self.attributeNames[alias] = attr
        self.attributeNeeded[alias] = attr
        return attr, Aggregation.NONE

    def visitAttributeAggr(self, ctx:miniSQLParser.AttributeAggrContext):
        print_debug("visitAttributeAggr")
        attr = self.visit(ctx.att())
        self.attributeNames[attr.fullName] = attr
        self.attributeNeeded[attr.fullName] = attr
        self.hasAggreg = True
        aggr = ctx.aggr().getText()
        if aggr == "MAX":
            aggr = Aggregation.MAX
        elif aggr == "MIN":
            aggr = Aggregation.MIN
        elif aggr == "COUNT":
            aggr = Aggregation.COUNT
        elif aggr == "SUM":
            aggr = Aggregation.SUM
        return attr, aggr

    # __________________ att rule ______________________________________________
    def visitAtt(self, ctx:miniSQLParser.AttContext):
        print_debug("visitAtt")
        tableName = ctx.ID(0).getText()
        attrName = ctx.ID(1).getText()
        return Attribute(tableName,attrName)

    # __________________ rels rules ____________________________________________
    def visitRelationDeclList(self, ctx:miniSQLParser.RelationDeclListContext):
        print_debug("visitRelationDeclList")
        queueRel = self.visit(ctx.rels())
        firstRel = self.visit(ctx.rel())
        queueRel.add(firstRel)
        return queueRel

    def visitRelationDeclSimple(self, ctx:miniSQLParser.RelationDeclSimpleContext):
        print_debug("visitRelationDeclSimple")
        x = self.visit(ctx.rel())
        out = set()
        out.add(x) #Not juste set of x because it unpacks tuples...
        return out

    # __________________ rel rules _____________________________________________
    def visitRelationID(self, ctx:miniSQLParser.RelationIDContext):
        print_debug("visitRelationID")
        fileName = ctx.FILENAME().getText()
        tableName = ctx.ID().getText()
        return (fileName,tableName)

    def visitRelationSubQuery(self, ctx:miniSQLParser.RelationSubQueryContext):
        # When a table in the FROM is a subquery
        print_debug("visitRelationSubQuery")
        tableName = ctx.ID().getText()

        mem = self.allAttr
        self.allAttr = False
        rel = self.visit(ctx.sql())
        self.allAttr = mem

        self.relationNames[tableName]=tableName
        rel.rename(tableName)
        self.dataManager.add_table(rel)

        return (tableName,tableName)

    # _____________________ cond rules _________________________________________
    def visitCondOrList(self, ctx:miniSQLParser.CondOrListContext):
        print_debug("visitCondOrList")
        queueCond, setRel1, notIn1 = self.visit(ctx.cond())
        firstCond, setRel2, notIn2 = self.visit(ctx.and_cond())
        return [And(firstCond)]+queueCond, (setRel1 | setRel2), (notIn1 or notIn2)

    def visitCondOrSimple(self, ctx:miniSQLParser.CondOrSimpleContext):
        print_debug("visitCondOrSimple")
        cond , setRel, notIn = self.visit(ctx.and_cond())
        return [And(cond)], setRel, notIn

    # ____________________ and_cond rules ______________________________________
    def visitCondAndOr(self, ctx:miniSQLParser.CondAndOrContext):
        print_debug("visitCondAndOr")
        queueCond, setRel1, notIn1 = self.visit(ctx.and_cond())
        firstCond, setRel2, notIn2 = self.visit(ctx.cond())
        return [Or(firstCond)]+queueCond, (setRel1 | setRel2), (notIn1 or notIn2)

    def visitCondAndList(self, ctx:miniSQLParser.CondAndListContext):
        print_debug("visitCondAndList")
        queueCond, setRel1, notIn1 = self.visit(ctx.and_cond())
        firstCond, setRel2, notIn2 = self.visit(ctx.at_cond())
        return firstCond+queueCond, (setRel1 | setRel2), (notIn1 or notIn2)

    def visitCondAndSimple(self, ctx:miniSQLParser.CondAndSimpleContext):
        print_debug("visitCondAndSimple")
        cond, rel, notIn = self.visit(ctx.at_cond())
        return (cond, rel, notIn)

    # ____________________ at_cond rules _______________________________________
    def visitCompSimple(self, ctx:miniSQLParser.CompSimpleContext):
        print_debug("visitCompSimple")
        op = text_to_OP(ctx.op().getText())
        if ctx.CONST() is not None :
            attr,const = self.visit(ctx.att(0)), ctx.CONST().getText()
            self.attributeNeeded[attr.fullName] = attr
            return [ConstCondition(attr,op,const)], set(), False
        else:
            attr1,attr2 = self.visit(ctx.att(0)), self.visit(ctx.att(1))
            self.attributeNeeded[attr1.fullName] = attr1
            self.attributeNeeded[attr2.fullName] = attr2
            return [Condition(attr1,op,attr2)], set(), False

    def visitCompIn(self, ctx:miniSQLParser.CompInContext):
        print_debug("visitCompIn")
        attr = self.visit(ctx.att())
        self.attributeNeeded[attr.fullName] = attr
        cond, rel = self.visitSubSql(ctx.sql(), attr)
        return cond, rel, False

    def visitCompNotIn(self, ctx:miniSQLParser.CompNotInContext):
        print_debug("visitCompNotIn")
        attr = self.visit(ctx.att())
        self.attributeNeeded[attr.fullName] = attr
        cond,rel = self.visitSubSql(ctx.sql(), attr)
        return [NotInCond(cond)], rel, True


import urllib.request
import urllib.parse
import json
import os
trans = input('Please type in what you want to be checked:')
params = urllib.parse.urlencode({'a': 'getWordMean', 'c': 'search', 'word': trans})
url = "http://www.iciba.com/index.php?%s" % params
response = urllib.request.urlopen(url)
html = response.read().decode('utf-8')
results = json.loads(html)
check=results['baesInfo']
check_result='suggest' in check.keys()
if check_result == True:
    corword=results['baesInfo']['suggest'][0]['key']
    print( 0)
else:
    print( 1)