# -*- coding:utf-8 -*-
# ==========================================
#       author: ZiChen
#        mail: 1538185121@qq.com
#         time: 2021/05/03
#           歌词下载脚本
# ==========================================

# 请求及数据处理库
import re
from urllib import request
import json
import traceback
import os
# 本地API
import QQMusicAPI  # 本地QQ音乐API
# 输出格式设置
from datetime import datetime
version = '0.2.0'

# 更新日志
# 2021/06/20 🔧更改程序架构，优化程序执行顺序，
# 2021/06/19 🎵增加对QQ音乐单曲歌词下载支持

def urlProcessing(songUrl):
    '''
    将输入的歌曲链接进行处理得到想要的歌曲链接

    songUrl 歌曲链接，如示例（网易云）
    '''
    Log = '[{levelname}] - {funcName} - '.format(levelname='DEBUG',
                                                 funcName='urlProcessing')

    Log_ERROR = '[{levelname}] - {funcName} - '.format(levelname='ERROR',
                                                       funcName='urlProcessing')

    if type(songUrl) == list:  # 如果传入的是列表，即需要下载的歌单歌曲
        Type = 'PlayList_download'
    # 通过分析链接识别歌曲或歌单来源
    elif type(songUrl) == str:
        # 2021/06/20 先判断是歌曲|歌单歌曲获取
        Type = 'Song_PlayListCheck'
        if re.search(r'music.163.com/song', songUrl) != None:  # 网易云单曲
            Type = Type + '|Netease_Song'
        elif re.search(r'music.163.com/#/playlist|music.163.com/playlist', songUrl) != None:  # 网易云歌单
            Type = Type + '|Netease_PlayList_check'
        elif re.search(r'y.qq.com/n/ryqq/songDetail', songUrl) != None:  # 2021/06/19 QQ音乐单曲
            Type = Type + '|QQ_Music_Song'

    if Type.split('|')[0] == 'Song_PlayListCheck':  # 2021/06/20 确认为歌曲|歌单歌曲获取

        # 确认后获取歌曲所属平台做后续处理
        Type = Type.split('|')[-1]

        if Type == 'QQ_Music_Song':  # QQ音乐;调用本地API获取歌词及歌曲信息
            print(str(datetime.today()).split(' ')[
                1].split('.')[0]+Log+'识别到QQ音乐歌曲')
            print(str(datetime.today()).split(' ')[
                1].split('.')[0]+Log+'正在处理链接...')
            # 2021/06/19 QQ音乐单曲的mid就在url的最后
            songID = songUrl.split('/')[-1]
            print(str(datetime.today()).split(' ')[
                1].split('.')[0]+Log+'已获取歌曲id：%s' % songID)

            try:
                # 获得歌手名-歌曲名，用于歌词写入
                print(str(datetime.today()).split(' ')[
                    1].split('.')[0]+Log+'正在获取歌曲信息...')

                myjson_detail = QQMusicAPI.Details_GET(songID)
                # 从字典中获得歌曲的名字及作者/翻唱者
                songName = myjson_detail['name']
                # 由于作者/翻唱者可能有多个故使用列表存储，最后用join拼接即可
                songAuthor = myjson_detail['ar']

                # 由于作者/翻唱者之间用 / 隔开会导致文件命名时出错故将 / 替换成 , 但这样做也会使下载的歌曲文件
                # 无法正确被播放器识别，暂时的解决方法是给出提示让用户自己去改名
                if bool(re.search(r'[/]', songAuthor)) == True:
                    print(str(datetime.today()).split(' ')[1].split(
                        '.')[0]+Log_ERROR+'%s 【歌曲名称错误！下载歌词文件后请自行更改歌词文件名！】' % songAuthor)
                    songAuthor = songAuthor.replace('/', ',')

                songDetail = '%s - %s' % (songAuthor, songName)
                print(str(datetime.today()).split(' ')[1].split(
                    '.')[0]+Log+'已获取歌曲信息: %s\n' % songDetail)

                # 获得歌词文本
                print(str(datetime.today()).split(' ')
                      [1].split('.')[0]+Log+'发送请求中...')

                print(str(datetime.today()).split(' ')[
                    1].split('.')[0]+Log+'正在处理接受的数据...')
                # 从字典中获得歌词文本
                lyrics = QQMusicAPI.Lyrics_GET(songID)['lyric']

                print(str(datetime.today()).split(' ')[
                    1].split('.')[0]+Log+'数据处理完毕，已取得歌词文本√\n')

                print(str(datetime.today()).split(' ')[
                    1].split('.')[0]+Log+'正在将歌词写入文件...')
                with open('./%s.lrc' % songDetail, 'w', encoding='utf-8') as f:
                    f.write(lyrics)

                print(str(datetime.today()).split(' ')
                      [1].split('.')[0]+Log+'已保存歌词文件√\n')

                # 随便返回个东西
                return True

            except:
                traceback.print_exc()
                print(str(datetime.today()).split(' ')
                      [1].split('.')[0]+Log+'错误！正在重试...\n')
                urlProcessing(songUrl)

        else:  # QQ音乐无法通过get方法获得，得调用本地api获取
            if Type == 'Netease_Song':  # 网易云
                patternID = re.compile(r'[id=]\d+[&]')  # 查找数字
                print(str(datetime.today()).split(' ')[
                    1].split('.')[0]+Log+'识别到网易云音乐歌曲')
                songID = re.sub(r'[=]|[&]', '', patternID.findall(songUrl)[0])
                # 网易云音乐歌词api
                neteaseApiUrl_lyric = 'https://zichen-cloud-music-api.vercel.app/lyric?id=%s&realIP=116.25.146.177' % songID
                # 网易云音乐歌曲信息api
                neteaseApiUrl_detail = 'https://zichen-cloud-music-api.vercel.app/song/detail?ids=%s' % songID

            try:
                # 获得歌手名-歌曲名，用于歌词写入
                print(str(datetime.today()).split(' ')[
                    1].split('.')[0]+Log+'正在获取歌曲信息...')

                if Type == 'Netease_Song':  # 网易云
                    req_detail = request.Request(url=neteaseApiUrl_detail)
                res_detail = request.urlopen(req_detail)
                # 获取响应的json字符串
                str_json_detail = res_detail.read().decode('utf-8')
                # 把json转换成字典
                myjson_detail = json.loads(str_json_detail)
                # 从字典中获得歌曲的名字及作者/翻唱者
                if Type == 'Netease_Song':  # 网易云
                    songName = myjson_detail['songs'][0]['name']
                    # 由于作者/翻唱者可能有多个故使用列表存储，最后用join拼接即可
                    songAuthorLst = []
                    for i in myjson_detail['songs'][0]['ar']:
                        songAuthorLst.append(i['name'])

                # 由于作者/翻唱者之间用 / 隔开会导致文件命名时出错故将 / 替换成 , 但这样做也会使下载的歌曲文件
                # 无法正确被播放器识别，暂时的解决方法是给出提示让用户自己去改名
                if bool(re.search(r'[/]', i['name'])) == True:
                    print(str(datetime.today()).split(' ')[1].split(
                        '.')[0]+Log_ERROR+'%s 【歌曲名称错误！下载歌词文件后请自行更改歌词文件名！】' % i['name'])
                songAuthor = re.sub(
                    r'[/]', ',', ','.join(songAuthorLst))
                songDetail = '%s - %s' % (songAuthor, songName)
                print(str(datetime.today()).split(' ')[1].split(
                    '.')[0]+Log+'已获取歌曲信息: %s\n' % songDetail)

                # 获得歌词文本
                print(str(datetime.today()).split(' ')
                      [1].split('.')[0]+Log+'发送请求中...')

                if Type == 'Netease_Song':  # 网易云
                    req_lyric = request.Request(url=neteaseApiUrl_lyric)

                res_lyric = request.urlopen(req_lyric)
                print(str(datetime.today()).split(' ')
                      [1].split('.')[0]+Log+'已接收数据√')
                # 获取响应的json字符串
                str_json_lyric = res_lyric.read().decode('utf-8')
                # 把json转换成字典
                myjson_lyric = json.loads(str_json_lyric)
                print(str(datetime.today()).split(' ')[
                    1].split('.')[0]+Log+'正在处理接受的数据...')
                # 从字典中获得歌词文本
                if Type == 'Netease_Song':  # 网易云
                    lyrics = myjson_lyric['lrc']['lyric']

                print(str(datetime.today()).split(' ')[
                    1].split('.')[0]+Log+'数据处理完毕，已取得歌词文本√\n')
                # print(lyrics+'\n')

                print(str(datetime.today()).split(' ')[
                    1].split('.')[0]+Log+'正在将歌词写入文件...')
                with open('./%s.lrc' % songDetail, 'w', encoding='utf-8') as f:
                    f.write(lyrics)

                print(str(datetime.today()).split(' ')
                      [1].split('.')[0]+Log+'已保存歌词文件√\n')

                # 随便返回个东西
                return True
            except:
                traceback.print_exc()
                print(str(datetime.today()).split(' ')
                      [1].split('.')[0]+Log+'错误！正在重试...\n')
                urlProcessing(songUrl)

    elif Type == 'PlayList_check':
        # 歌单查看并返回歌单详情
        try:
            if Type == 'Netease_Song':  # 网易云
                print(str(datetime.today()).split(' ')[
                    1].split('.')[0]+Log+'识别到网易云音乐歌单')

            print(str(datetime.today()).split(' ')[
                1].split('.')[0]+Log+'正在处理链接...')
            patternID = re.compile(r'[id=]\d+[&]')  # 查找数字

            playListID = re.sub(
                r'[=]|[&]', '', patternID.findall(songUrl)[0])
            print(str(datetime.today()).split(' ')[
                1].split('.')[0]+Log+'已获取歌单id：%s' % playListID)

            limit = 1001  # 歌单中歌曲信息获取数量限制

            # 网易云音乐歌单详细信息api
            neteaseApiUrl_playList = 'https://zichen-cloud-music-api.vercel.app/playlist/detail?id=%s' % playListID
            print(str(datetime.today()).split(' ')[
                1].split('.')[0]+Log+'正在向：[%s] 获取歌单信息...' % neteaseApiUrl_playList)

            # 加标头
            header = {
                "User-Agent": "mozilla/4.0 (compatible; MSIE 5.5; Windows NT)",
            }

            req_playList = request.Request(
                url=neteaseApiUrl_playList, headers=header)
            res_playList = request.urlopen(req_playList)

            # 获取响应的json字符串
            str_json_playList = res_playList.read().decode('utf-8')
            # 把json转换成字典
            myjson_playList = json.loads(str_json_playList)

            # 逐个获取歌单内的歌曲名及相应作者/翻唱者
            songList = []
            # 用于计数显示当前过程的数字
            start_num = 0
            total_num = len(
                myjson_playList["playlist"]["trackIds"])  # 总歌单歌曲数

            # 根据大佬所述，未登录状态下无法获取歌单完整曲目，但trackIds是完整的，故获取trackIds后逐个请求，但此方法效率较低
            for songTotal in myjson_playList["playlist"]["trackIds"]:
                songID = songTotal['id']  # 获得歌曲id

                # 网易云音乐歌词api
                neteaseApiUrl_lyric = 'https://zichen-cloud-music-api.vercel.app/lyric?id=%s&realIP=116.25.146.177' % songID
                # 网易云音乐歌曲信息api
                neteaseApiUrl_detail = 'https://zichen-cloud-music-api.vercel.app/song/detail?ids=%s' % songID

                req_detail = request.Request(url=neteaseApiUrl_detail)
                res_detail = request.urlopen(req_detail)
                # 获取响应的json字符串
                str_json_detail = res_detail.read().decode('utf-8')
                # 把json转换成字典
                myjson_detail = json.loads(str_json_detail)
                # 从字典中获得歌曲的名字及作者/翻唱者
                # Tip：由于获取的歌曲名有\xa0不间断符号故使用join+split消除该符号
                songName = "" .join(
                    myjson_detail['songs'][0]['name'].split())
                # 由于作者/翻唱者可能有多个故使用列表存储，最后用join拼接即可
                songAuthorLst = []
                for i in myjson_detail['songs'][0]['ar']:
                    songAuthorLst.append(i['name'])
                # 由于作者/翻唱者之间用 / 隔开会导致文件命名时出错故将 / 替换成 , 但这样做也会使下载的歌曲文件
                # 无法正确被播放器识别，暂时的解决方法是给出提示让用户自己去改名
                    if bool(re.search(r'[/]', i['name'])) == True:
                        print(str(datetime.today()).split(' ')[1].split(
                            '.')[0]+Log_ERROR+'%s 【歌曲名称错误！下载歌词文件后请自行更改歌词文件名！】' % i['name'])

                songAuthor = re.sub(
                    r'[/]', ',', ','.join(songAuthorLst))
                # 将 作者/翻唱者+歌曲名+歌曲ID 用元组形式存储并最终存储至列表中
                # [(歌曲1),(歌曲2),...]
                songList.append([songAuthor, songName, str(songID)])
                # 显示完成情况，用print覆盖打印
                start_num += 1
                print('\r歌单歌曲读取已完成(%s/%s)' %
                      (start_num, total_num), end='')

            print('\n'+str(datetime.today()).split(' ')[
                1].split('.')[0]+Log+'已获取歌单信息√\n')
            for i in songList:
                print('%s - %s - ID:%s' % (i[0], i[1], i[2]))
            print('\n'+'-'*15)

            return songList
        except:  # 错误重试
            traceback.print_exc()
            print(str(datetime.today()).split(' ')
                  [1].split('.')[0]+Log+'错误！正在重试...\n')
            urlProcessing(songUrl)

    elif Type == 'PlayList_download':
        # 歌单歌曲下载，传入的songID

        print(str(datetime.today()).split(' ')[
            1].split('.')[0]+Log+'正在启动批量下载模块...')

        # 用于计数显示当前过程的数字
        start_num = 0
        total_num = len(songUrl)  # 总歌单歌曲数

        # 先解包
        for songLst in songUrl:
            songDetail = '%s - %s' % (songLst[0], songLst[1])
            songID = songLst[2]
            # print('songID:%s' % songID)
            # print('songLst=%s\n' % songLst)
            start_num += 1
            # 开始下载
            # 网易云音乐歌词api
            neteaseApiUrl_lyric = 'https://zichen-cloud-music-api.vercel.app/lyric?id=%s&realIP=116.25.146.177' % songID
            # print(neteaseApiUrl_lyric)

            # 出错后会重新循环，跳过已经保存的文件，提升效率，避免重复请求
            if os.path.exists('./%s.lrc' % songDetail) == True:
                pass
            else:
                try:
                    # 获得歌词文本
                    req_lyric = request.Request(url=neteaseApiUrl_lyric)
                    res_lyric = request.urlopen(req_lyric)
                    # 获取响应的json字符串
                    str_json_lyric = res_lyric.read().decode('utf-8')
                    # 把json转换成字典
                    myjson_lyric = json.loads(str_json_lyric)
                    # 从字典中获得歌词文本
                    lyrics = myjson_lyric['lrc']['lyric']

                    with open('./%s.lrc' % songDetail, 'w', encoding='utf-8') as f:
                        f.write(lyrics)

                    print('\r已下载(%s/%s)' % (start_num, total_num), end='')
                    if start_num == total_num:  # 下载完提示
                        print('\n'+str(datetime.today()).split(' ')
                              [1].split('.')[0]+Log+'歌单歌曲歌词下载完毕√')

                except:
                    # traceback.print_exc()
                    print(str(datetime.today()).split(' ')
                          [1].split('.')[0]+Log+'{%s}下载错误！\n已跳过出错的歌曲链接\n' % songDetail)
                    # 删除出错的元素
                    # print('songUrl=%s\n' % songUrl)
                    del songUrl[start_num-1]
                    # print('songUrl=%s\n' % songUrl)
                    # print(type(songUrl))
                    if start_num == total_num:  # 下载完提示
                        print('\n'+str(datetime.today()).split(' ')
                              [1].split('.')[0]+Log+'歌单歌曲歌词下载完毕√')
                    else:
                        urlProcessing(songUrl)


from config import Config
import requests, json
from app.models import news_article, news_source

MOVIE_API_KEY = Config.API_KEY
News_Article = news_article.Article
News_Source = news_source.Source

def configure_request(app):
    global api_key
    api_key = app.config['API_KEY']

def get_news():
    request = requests.get('https://newsapi.org/v2/everything?q=all&apiKey={}'
                           .format(MOVIE_API_KEY))
    response = json.loads(request.content)
    news = []
    for new in response['articles']:
        new = News_Article(new['source'], new['author'], new['title'], new['description'], new['urlToImage'],
                           new['url'], new['publishedAt'])
        news.append(new)

    return news

def get_news_sources():
    request = requests.get('https://newsapi.org/v2/top-headlines/sources?apiKey={}'
                           .format(MOVIE_API_KEY))
    response = json.loads(request.content)
    news_sources = []
    for source in response['sources']:
        source = News_Source(source['id'], source['name'])
        news_sources.append(source)

    return news_sources

def get_news_from_source(source):
    request = requests.get('https://newsapi.org/v2/everything?q={}&apiKey={}'.format(source, MOVIE_API_KEY))
    response = json.loads(request.content)
    news = []
    for new in response['articles']:
        new = News_Article(new['source'], new['author'], new['title'], new['description'], new['urlToImage'],
                           new['url'], new['publishedAt'])
        news.append(new)
    return news


from flask import Flask, render_template, request
from transformers import VisionEncoderDecoderModel, ViTFeatureExtractor, AutoTokenizer
import torch
from PIL import Image
import io
import base64

app = Flask(__name__)

model = VisionEncoderDecoderModel.from_pretrained("nlpconnect/vit-gpt2-image-captioning")
feature_extractor = ViTFeatureExtractor.from_pretrained("nlpconnect/vit-gpt2-image-captioning")
tokenizer = AutoTokenizer.from_pretrained("nlpconnect/vit-gpt2-image-captioning")

device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
model.to(device)

max_length = 16
num_beams = 4
gen_kwargs = {"max_length": max_length, "num_beams": num_beams}

@app.route('/', methods=['GET', 'POST'])
def index():
    if request.method == 'POST':
        # Check if a file was uploaded
        if 'image' not in request.files:
            return render_template('index.html', error='No image uploaded')

        file = request.files['image']

        # Check if the file has a valid extension
        if file.filename == '':
            return render_template('index.html', error='No image selected')

        if file and allowed_file(file.filename):
            # Save the uploaded image
            file_path = 'uploads/' + file.filename
            file.save(file_path)

            # Generate captions using the uploaded image
            captions = predict_step([file_path])

            # Display the image using PIL
            image = Image.open(file_path)
            image_data = io.BytesIO()
            image.save(image_data, format='PNG')
            image_base64 = base64.b64encode(image_data.getvalue()).decode('utf-8')

            return render_template('index.html', image=image_base64, captions=captions)
        else:
            return render_template('index.html', error='Invalid file type')

    return render_template('index.html')

def allowed_file(filename):
    # Add the allowed image file extensions here
    ALLOWED_EXTENSIONS = {'png', 'jpg', 'jpeg'}
    return '.' in filename and filename.rsplit('.', 1)[1].lower() in ALLOWED_EXTENSIONS

def predict_step(image_paths):
    images = []
    for image_path in image_paths:
        i_image = Image.open(image_path)
        if i_image.mode != "RGB":
            i_image = i_image.convert(mode="RGB")
        images.append(i_image)

    pixel_values = feature_extractor(images=images, return_tensors="pt").pixel_values
    pixel_values = pixel_values.to(device)

    output_ids = model.generate(pixel_values, num_return_sequences=3, **gen_kwargs)

    preds = tokenizer.batch_decode(output_ids, skip_special_tokens=True)
    preds = [pred.strip() for pred in preds]
    return preds

if __name__ == '__main__':
    app.run(debug=True)


import requests
from lxml import etree
import os

'''if __name__=='__main__':
    try:
        url='https://pic.netbian.com/4kmeinv/'
        headers={'user-agent':'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/95.0.4638.69 Safari/537.36'}
        response=requests.get(url=url,headers=headers)
        response.raise_for_status()
        response.encoding=response.apparent_encoding
        response_text=response.text
        print(response_text)
    except:
        print('网络连接异常')'''

import requests
from lxml import etree
import os
if __name__=='__main__':
    url='https://pic.netbian.com/4kmeinv/'
    headers={'user-agent':'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/95.0.4638.69 Safari/537.36'}
    response=requests.get(url=url,headers=headers)
    #可以手动修改响应数据的编码格式
    #response.encoding='gbk'
    page_text=response.text
    print(page_text)

    tree=etree.HTML(page_text)
    list_li=tree.xpath('//div[@class="slist"]//li')
    print(list_li)
    #创建一个文件夹
    if not os.path.exists('./piclibs'):
        os.mkdir('./piclibs')

    for li in list_li:
        img_src='https://pic.netbian.com'+li.xpath('./a/img/@src')[0]
        img_name=li.xpath('./a/img/@alt')[0]+'.jpg'
        #通用处理中文乱码的方法(注意重新赋值）
        img_name=img_name.encode('iso-8859-1').decode('gbk')
        #print(img_name,img_src)
        #请求图片进行持久存储
        img_data=requests.get(url=img_src,headers=headers).content
        img_path='piclibs/'+img_name
        with open(img_path,'wb') as fp:
            fp.write(img_data)
            print(img_name,'下载成功!')

7.xpath解析案例－下载图片数据.py

7.xpath解析案例－下载图片数据.py


#!/usr/bin/env python
# -*- coding: utf-8 -*-

import netCDF4
from utils import *

def write_jules_overbank_props_1d(overbank_fn, overbank_maps, grid_dim_name):

    nco = netCDF4.Dataset(overbank_fn, 'w', format='NETCDF4')
    mask = LAND_FRAC > 0.
    nland = mask.sum()
    for key, value in overbank_maps.items():
        overbank_maps[key] = value.transpose()[mask.transpose()]
    nco.createDimension(grid_dim_name, nland)
    var = nco.createVariable(
        'logn_mean', 'f8', (grid_dim_name,), fill_value=F8_FILLVAL
    )
    var.units = 'ln(m)'
    var.standard_name = 'logn_mean'
    var[:] = overbank_maps['logn_mean']

    var = nco.createVariable(
        'logn_stdev', 'f8', (grid_dim_name,), fill_value=F8_FILLVAL
    )
    var.units = 'ln(m)'
    var.standard_name = 'logn_stdev'
    var[:] = overbank_maps['logn_stdev']
    nco.close()

def write_jules_overbank_props_2d(overbank_fn, overbank_maps, x_dim_name, y_dim_name):

    nco = netCDF4.Dataset(overbank_fn, 'w', format='NETCDF4')
    nco = add_lat_lon_dims_2d(nco, x_dim_name, y_dim_name)
    var = nco.createVariable(
        'logn_mean', 'f8', (y_dim_name, x_dim_name),
        fill_value=F8_FILLVAL
    )
    var.units = 'ln(m)'
    var.standard_name = 'logn_mean'
    var.grid_mapping = 'latitude_longitude'
    var[:] = overbank_maps['logn_mean']

    var = nco.createVariable(
        'logn_stdev', 'f8', (y_dim_name, x_dim_name),
        fill_value=F8_FILLVAL
    )
    var.units = 'ln(m)'
    var.standard_name = 'logn_stdev'
    var.grid_mapping = 'latitude_longitude'
    var[:] = overbank_maps['logn_stdev']
    nco.close()

# def write_jules_overbank_props(overbank_fn, one_d=False):

#     # Read overbank properties:
#     logn_mean_ds = rasterio.open(os.environ['LOGN_MEAN_FN'])
#     logn_stdev_ds = rasterio.open(os.environ['LOGN_STDEV_FN'])
#     overbank_maps = {}
#     overbank_maps['logn_mean'] = logn_mean_ds.read(1, masked=False).squeeze()
#     overbank_maps['logn_stdev'] = logn_stdev_ds.read(1, masked=False).squeeze()
#     for var in overbank_maps.keys():
#         arr = overbank_maps[var]
#         arr = np.ma.masked_array(
#             arr,
#             mask=np.broadcast_to(
#                 np.logical_not(LAND_FRAC),
#                 arr.shape
#             ),
#             dtype=np.float64,
#             fill_value=F8_FILLVAL
#         )
#         overbank_maps[var] = arr

#     # Write netCDF:
#     if one_d:
#         write_jules_overbank_props_1d(overbank_fn, overbank_maps)
#     else:
#         write_jules_overbank_props_2d(overbank_fn, overbank_maps)


"""HTTP Archive dataflow pipeline for generating HAR data on BigQuery."""

from __future__ import absolute_import

import json
import logging
from copy import deepcopy
from hashlib import sha256

import apache_beam as beam

from modules import utils, constants, transformation

# BigQuery can handle rows up to 100 MB.
MAX_CONTENT_SIZE = 2 * 1024 * 1024
# Number of times to partition the requests tables.
NUM_PARTITIONS = 4

def get_page(har):
    """Parses the page from a HAR object."""

    if not har:
        return None

    page = har.get("log").get("pages")[0]
    url = page.get("_URL")

    metadata = get_metadata(har)
    if metadata:
        # The page URL from metadata is more accurate.
        # See https://github.com/HTTPArchive/data-pipeline/issues/48
        url = metadata.get("tested_url", url)

    try:
        page = trim_page(page)
        payload_json = to_json(page)
    except Exception:
        logging.warning(
            'Skipping pages payload for "%s": unable to stringify as JSON.' % url
        )
        return None

    payload_size = len(payload_json)
    if payload_size > MAX_CONTENT_SIZE:
        logging.warning(
            'Skipping pages payload for "%s": payload size (%s) exceeds the maximum content size of %s bytes.'
            % (url, payload_size, MAX_CONTENT_SIZE)
        )
        return None

    return [
        {
            "url": url,
            "payload": payload_json,
            "date": har["date"],
            "client": har["client"],
            "metadata": metadata,
        }
    ]

def get_page_url(har):
    """Parses the page URL from a HAR object."""

    page = get_page(har)

    if not page:
        logging.warning("Unable to get URL from page (see preceding warning).")
        return None

    return page[0].get("url")

def get_metadata(har):
    page = har.get("log").get("pages")[0]
    metadata = page.get("_metadata")
    return metadata

def is_home_page(mapped_har):
    if not mapped_har:
        return False
    metadata = mapped_har.get("metadata")
    if metadata and "crawl_depth" in metadata:
        return metadata.get("crawl_depth") == 0
        # Only home pages have a crawl depth of 0.
    else:
        return True
        # legacy default

def partition_step(har, num_partitions):
    """Returns a partition number based on the hashed HAR page URL"""

    if not har:
        logging.warning("Unable to partition step, null HAR.")
        return 0

    page_url = get_page_url(har)

    if not page_url:
        logging.warning("Skipping HAR: unable to get page URL (see preceding warning).")
        return 0

    _hash = hash_url(page_url)

    # shift partitions by one so the zero-th contains errors
    offset = 1

    return (_hash % (num_partitions - 1)) + offset

def get_requests(har):
    """Parses the requests from a HAR object."""

    if not har:
        return None

    page_url = get_page_url(har)

    if not page_url:
        # The page_url field indirectly depends on the get_page function.
        # If the page data is unavailable for whatever reason, skip its requests.
        logging.warning(
            "Skipping requests payload: unable to get page URL (see preceding warning)."
        )
        return None

    entries = har.get("log").get("entries")

    requests = []

    for request in entries:

        request_url = request.get("_full_url")

        if not request_url:
            logging.warning('Skipping empty request URL for "%s"', page_url)
            continue

        try:
            payload = to_json(trim_request(request))
        except Exception:
            logging.warning(
                'Skipping requests payload for "%s": unable to stringify as JSON.'
                % request_url
            )
            continue

        payload_size = len(payload)
        if payload_size > MAX_CONTENT_SIZE:
            logging.warning(
                'Skipping requests payload for "%s": payload size (%s) exceeded maximum content size of %s bytes.'
                % (request_url, payload_size, MAX_CONTENT_SIZE)
            )
            continue

        metadata = get_metadata(har)

        requests.append(
            {
                "page": page_url,
                "url": request_url,
                "payload": payload,
                "date": har["date"],
                "client": har["client"],
                "metadata": metadata,
            }
        )

    return requests

def trim_request(request):
    """Removes redundant fields from the request object."""

    # Make a copy first so the response body can be used later.
    request = deepcopy(request)
    request.get("response").get("content").pop("text", None)
    return request

def trim_page(page):
    """Removes unneeded fields from the page object."""

    if not page:
        return None

    # Make a copy first so the data can be used later.
    page = deepcopy(page)
    page.pop("_parsed_css", None)
    return page

def hash_url(url):
    """Hashes a given URL to a process-stable integer value."""
    return int(sha256(url.encode("utf-8")).hexdigest(), 16)

def get_response_bodies(har):
    """Parses response bodies from a HAR object."""

    page_url = get_page_url(har)
    requests = har.get("log").get("entries")

    response_bodies = []

    for request in requests:
        request_url = request.get("_full_url")
        body = None
        if request.get("response") and request.get("response").get("content"):
            body = request.get("response").get("content").get("text", None)

        if body is None:
            continue

        truncated = len(body) > MAX_CONTENT_SIZE
        if truncated:
            logging.warning(
                'Truncating response body for "%s". Response body size %s exceeds limit %s.'
                % (request_url, len(body), MAX_CONTENT_SIZE)
            )

        metadata = get_metadata(har)

        response_bodies.append(
            {
                "page": page_url,
                "url": request_url,
                "body": body[:MAX_CONTENT_SIZE],
                "truncated": truncated,
                "date": har["date"],
                "client": har["client"],
                "metadata": metadata,
            }
        )

    return response_bodies

def get_technologies(har):
    """Parses the technologies from a HAR object."""

    if not har:
        return None

    page = har.get("log").get("pages")[0]
    page_url = page.get("_URL")
    app_names = page.get("_detected_apps", {})
    categories = page.get("_detected", {})
    metadata = get_metadata(har)

    # When there are no detected apps, it appears as an empty array.
    if isinstance(app_names, list):
        app_names = {}
        categories = {}

    app_map = {}
    app_list = []
    for app, info_list in app_names.items():
        if not info_list:
            continue
        # There may be multiple info values. Add each to the map.
        for info in info_list.split(","):
            app_id = "%s %s" % (app, info) if len(info) > 0 else app
            app_map[app_id] = app

    for category, apps in categories.items():
        for app_id in apps.split(","):
            app = app_map.get(app_id)
            info = ""
            if app is None:
                app = app_id
            else:
                info = app_id[len(app):].strip()
            app_list.append(
                {
                    "url": page_url,
                    "category": category,
                    "app": app,
                    "info": info,
                    "date": har["date"],
                    "client": har["client"],
                    "metadata": metadata,
                }
            )

    return app_list

def get_lighthouse_reports(har):
    """Parses Lighthouse results from a HAR object."""

    if not har:
        return None

    report = har.get("_lighthouse")

    if not report:
        return None

    page_url = get_page_url(har)

    if not page_url:
        logging.warning(
            "Skipping lighthouse report: unable to get page URL (see preceding warning)."
        )
        return None

    # Omit large UGC.
    report.get("audits").get("screenshot-thumbnails", {}).get("details", {}).pop(
        "items", None
    )

    try:
        report_json = to_json(report)
    except Exception:
        logging.warning(
            'Skipping Lighthouse report for "%s": unable to stringify as JSON.'
            % page_url
        )
        return None

    report_size = len(report_json)
    if report_size > MAX_CONTENT_SIZE:
        logging.warning(
            'Skipping Lighthouse report for "%s": Report size (%s) exceeded maximum content size of %s bytes.'
            % (page_url, report_size, MAX_CONTENT_SIZE)
        )
        return None

    metadata = get_metadata(har)

    return [
        {
            "url": page_url,
            "report": report_json,
            "date": har["date"],
            "client": har["client"],
            "metadata": metadata,
        }
    ]

def get_parsed_css(har):
    """Extracts the parsed CSS custom metric from the HAR."""

    if not har:
        return None

    page = har.get("log").get("pages")[0]
    page_url = get_page_url(har)

    if not page_url:
        logging.warning("Skipping parsed CSS, no page URL")
        return None

    metadata = get_metadata(har)
    if metadata:
        page_url = metadata.get("tested_url", page_url)

    is_root_page = True
    if metadata:
        is_root_page = metadata.get("crawl_depth") == 0

    custom_metric = page.get("_parsed_css")

    if not custom_metric:
        logging.warning("No parsed CSS data for page %s", page_url)
        return None

    parsed_css = []

    for entry in custom_metric:
        url = entry.get("url")
        ast = entry.get("ast")

        if url == 'inline':
            # Skip inline styles for now. They're special.
            continue

        try:
            ast_json = to_json(ast)
        except Exception:
            logging.warning(
                'Unable to stringify parsed CSS to JSON for "%s".'
                % page_url
            )
            continue

        parsed_css.append({
            "date": har["date"],
            "client": har["client"],
            "page": page_url,
            "is_root_page": is_root_page,
            "url": url,
            "css": ast_json
        })

    return parsed_css

def to_json(obj):
    """Returns a JSON representation of the object.

    This method attempts to mirror the output of the
    legacy Java Dataflow pipeline. For the most part,
    the default `json.dumps` config does the trick,
    but there are a few settings to make it more consistent:

    - Omit whitespace between properties
    - Do not escape non-ASCII characters (preserve UTF-8)

    One difference between this Python implementation and the
    Java implementation is the way long numbers are handled.
    A Python-serialized JSON string might look like this:

        "timestamp":1551686646079.9998

    while the Java-serialized string uses scientific notation:

        "timestamp":1.5516866460799998E12

    Out of a sample of 200 actual request objects, this was
    the only difference between implementations. This can be
    considered an improvement.
    """

    if not obj:
        raise ValueError

    return json.dumps(obj, separators=(",", ":"), ensure_ascii=False)

def from_json(file_name, element):
    """Returns an object from the JSON representation."""

    try:
        return [(file_name, json.loads(element))]
    except Exception as e:
        logging.error('Unable to parse file %s into JSON object "%s...": %s' % (file_name, element[:50], e))
        return None

def add_date_and_client(element):
    """Adds `date` and `client` attributes to facilitate BigQuery table routing"""

    if element is None:
        logging.error('Element is empty, skipping adding date and time')
        return None

    try:
        file_name, har = element
        date, client = utils.date_and_client_from_file_name(file_name)
        page = har.get("log").get("pages")[0]
        metadata = page.get("_metadata", {})
        har.update(
            {
                "date": "{:%Y_%m_%d}".format(date),
                "client": metadata.get("layout", client).lower(),
            }
        )

        return har
    except Exception as e:
        logging.error('Unable to add date and client "%s...": %s' % (element[:50], e))
        return None

class WriteNonSummaryToBigQuery(beam.PTransform):
    def __init__(
        self,
        partitions,
        dataset_pages,
        dataset_technologies,
        dataset_lighthouse,
        dataset_requests,
        dataset_response_bodies,
        dataset_parsed_css,
        dataset_pages_home_only,
        dataset_technologies_home_only,
        dataset_lighthouse_home_only,
        dataset_requests_home_only,
        dataset_response_bodies_home_only,
        dataset_parsed_css_home_only,
        label=None,
        **kwargs,
    ):
        # TODO(BEAM-6158): Revert the workaround once we can pickle super() on py3.
        # super().__init__(label)
        beam.PTransform.__init__(self)
        self.label = label

        self.partitions = partitions
        self.dataset_pages = dataset_pages
        self.dataset_technologies = dataset_technologies
        self.dataset_lighthouse = dataset_lighthouse
        self.dataset_requests = dataset_requests
        self.dataset_response_bodies = dataset_response_bodies
        self.dataset_parsed_css = dataset_parsed_css
        self.dataset_pages_home = dataset_pages_home_only
        self.dataset_technologies_home = dataset_technologies_home_only
        self.dataset_lighthouse_home = dataset_lighthouse_home_only
        self.dataset_requests_home = dataset_requests_home_only
        self.dataset_response_bodies_home = dataset_response_bodies_home_only
        self.dataset_parsed_css_home = dataset_parsed_css_home_only

    def _transform_and_write_partition(
        self, pcoll, name, index, fn, table_all, table_home, schema
    ):
        formatted_name = utils.title_case_beam_transform_name(name)

        all_rows = pcoll | f"Map{formatted_name}{index}" >> beam.FlatMap(fn)

        home_only_rows = all_rows | f"Filter{formatted_name}{index}" >> beam.Filter(is_home_page)

        home_only_rows | f"Write{formatted_name}Home{index}" >> transformation.WriteBigQuery(
            table=lambda row: utils.format_table_name(row, table_home),
            schema=schema,
        )

    def expand(self, hars):
        # Add one to the number of partitions to use the zero-th partition for failures
        partitions = hars | beam.Partition(partition_step, self.partitions + 1)

        # log 0th elements (failures)
        partitions[0] | "LogPartitionFailures" >> beam.FlatMap(
            lambda e: logging.warning(f"Unable to partition record: {e}")
        )

        # enumerate starting from 1
        for idx in range(1, self.partitions + 1):
            self._transform_and_write_partition(
                pcoll=partitions[idx],
                name="pages",
                index=idx,
                fn=get_page,
                table_all=self.dataset_pages,
                table_home=self.dataset_pages_home,
                schema=constants.BIGQUERY["schemas"]["pages"],
            )

            self._transform_and_write_partition(
                pcoll=partitions[idx],
                name="technologies",
                index=idx,
                fn=get_technologies,
                table_all=self.dataset_technologies,
                table_home=self.dataset_technologies_home,
                schema=constants.BIGQUERY["schemas"]["technologies"],
            )

            self._transform_and_write_partition(
                pcoll=partitions[idx],
                name="lighthouse",
                index=idx,
                fn=get_lighthouse_reports,
                table_all=self.dataset_lighthouse,
                table_home=self.dataset_lighthouse_home,
                schema=constants.BIGQUERY["schemas"]["lighthouse"],
            )

            self._transform_and_write_partition(
                pcoll=partitions[idx],
                name="requests",
                index=idx,
                fn=get_requests,
                table_all=self.dataset_requests,
                table_home=self.dataset_requests_home,
                schema=constants.BIGQUERY["schemas"]["requests"],
            )

            self._transform_and_write_partition(
                pcoll=partitions[idx],
                name="response_bodies",
                index=idx,
                fn=get_response_bodies,
                table_all=self.dataset_response_bodies,
                table_home=self.dataset_response_bodies_home,
                schema=constants.BIGQUERY["schemas"]["response_bodies"],
            )

            self._transform_and_write_partition(
                pcoll=partitions[idx],
                name="parsed_css",
                index=idx,
                fn=get_parsed_css,
                table_all=self.dataset_parsed_css,
                table_home=self.dataset_parsed_css_home,
                schema=constants.BIGQUERY["schemas"]["parsed_css"],
            )


"""add region column for sample

Create Date: 2021-04-05 17:09:26.078925

"""
import enumtables  # noqa: F401
import sqlalchemy as sa
from alembic import op

# revision identifiers, used by Alembic.
revision = "20210405_170924"
down_revision = "20210401_211915"
branch_labels = None
depends_on = None

def upgrade():
    op.create_table(
        "region_types",
        sa.Column("item_id", sa.String(), nullable=False),
        sa.PrimaryKeyConstraint("item_id", name=op.f("pk_region_types")),
        schema="aspen",
    )
    op.enum_insert(
        "region_types",
        [
            "North America",
            "Oceania",
            "Asia",
            "Europe",
            "South America",
            "Africa",
        ],
        schema="aspen",
    )
    op.add_column(
        "samples",
        sa.Column(
            "region",
            sa.String(),
            nullable=True,
            comment="This is the continent this sample was collected from.",
        ),
        schema="aspen",
    )
    op.execute("""UPDATE aspen.samples SET region='North America'""")
    op.alter_column(
        "samples",
        "region",
        existing_type=sa.VARCHAR(),
        nullable=False,
        existing_comment="This is the continent this sample was collected from.",
        schema="aspen",
    )
    op.create_foreign_key(
        op.f("fk_samples_region_region_types"),
        "samples",
        "region_types",
        ["region"],
        ["item_id"],
        source_schema="aspen",
        referent_schema="aspen",
    )

def downgrade():
    op.drop_constraint(
        op.f("fk_samples_region_region_types"),
        "samples",
        schema="aspen",
        type_="foreignkey",
    )
    op.drop_column("samples", "region", schema="aspen")
    op.drop_table("region_types", schema="aspen")


#coding: utf-8
#
# example 11.4
#
import numpy as np
from geothermal_md import *
from  matplotlib.pyplot import *
from scipy.optimize import curve_fit
#
# donnees du probleme
#
gam = 0.5772157
M = np.loadtxt("..\\data\\pumping_test2.txt")
t = M[:,0]      # time in minutes
sf = M[:,1]     # drawndown in meters
nt = len(sf)
qo = 17      # heat transfer per metre
rw = 12.2
#
#
#
# first approach
#
Ti = 2.4     # m2/min
Si = 0.004
rbi = 0.01
G_vect = np.vectorize(leaky_function)
Si = 0.003
def s_theo(t,Tnew,Snew,rb):
    al = Tnew/Snew
    u = rw**2/(4*al*t)
    s = qo/(4*pi*Tnew)*G_vect(u,rb)
    return s
#
#
rbi = 0.03
po = [Ti,Si,rbi]
ni = 3
tn =t[ni:nt]
sn = sf[ni:nt]
params,resn = curve_fit(s_theo,tn,sn,po)
Tn = params[0]
Sn = params[1]
rbn = params[2]
print ('T = ',Tn,'m2/min')
print('S = ',Sn)
print('r/b = ',rbn)

s1 = s_theo(t,Ti,Si,rbi)
s2 = s_theo(t,Tn,Sn,rbn)
alh = Tn/Sn
u = rw**2/(4*alh*t)
un = rw**2/(4*alh*tn)
x = 1/u
p1 = loglog(x, sf, label='Measured',color = 'black')
p2 = loglog(x, s2, label='Curve fit',color = 'black',  linestyle='none', marker='o')
ll = legend()

sizeOfFont = 14
fontProperties = {'weight' : 'bold', 'size' : sizeOfFont}
a = gca()
gx = xlabel('1/u')
gy = ylabel('')
setp(gx,'fontsize',15,'fontweight','bold')
setp(gy,'fontsize',15,'fontweight','bold')
setp(a,'yscale','log')
setp(a,'xscale','log')
nx = len(x)
show()


# This is a sample Python script.
import pandas as pd
import csv
from datetime import datetime
import json
import paho.mqtt.client as mqtt
from itertools import count
import matplotlib.pyplot as plt
from matplotlib.animation import FuncAnimation

# Press Mayús+F10 to execute it or replace it with your code.
# Press Double Shift to search everywhere for classes, files, tool windows, actions, and settings.
field_names = ['timestamp', 'id', 'heart', 'accelerometer']
def write_on_csv(json_missatge):
    with open('base_dades_didac.csv', 'a') as csv_file:
        dict_object = csv.DictWriter(csv_file, fieldnames=field_names)
        dt=datetime.now()
        ts = datetime.timestamp(dt)
        new_entry = {'timestamp': int(ts)}
        new_entry.update(json_missatge)
        print("\nEl missatge rebut es:",new_entry)
        dict_object.writerow(new_entry)

def on_message(client, userdata, message):
    missatge_deco=str(message.payload.decode("utf-8"))
    print("el missatge es:",missatge_deco)
    #missatge = json.loads(missatge_deco)
    #write_on_csv(missatge)
    #print("message received ", str(message.payload.decode("utf-8")))

def subscribe_MQTT():
    client = mqtt.Client('SoftwareLazo')
    client.on_message = on_message
    client.connect('test.mosquitto.org')
    client.subscribe('SensorDidacLazo')
    client.loop_forever()

# Press the green button in the gutter to run the script.
if __name__ == '__main__':
    subscribe_MQTT()


#-*-coding:utf-8-*-
import argparse
import pyspark

from pyspark.sql.types import IntegerType
from pyspark.sql.functions import *

from generic_utils import execute_compute_stats

def extract_tbau_documento(spark):
    columns = [
		col("DOCU_DK").alias("DOAT_DOCU_DK"),
		col("DOCU_NR_EXTERNO").alias("DOAT_DOCU_NR_EXTERNO"),
		col("DOCU_NR_MP").alias("DOAT_DOCU_NR_MP"),
		col("DOCU_DT_CADASTRO").alias("DOAT_DOCU_DT_CADASTRO"),
		col("DOCU_DT_FATO").alias("DOAT_DOCU_DT_FATO"),
		col("DOCU_ORGI_ORGA_DK_RESPONSAVEL").alias("DOAT_ORGI_DK_RESPONSAVEL"),
		col("DOCU_ORGI_ORGA_DK_CARGA").alias("DOAT_ORGI_DK_CARGA"),
		col("DOCU_ORGA_DK_ORIGEM").alias("DOAT_ORGA_DK_ORIGEM"),
		col("DOCU_ORGE_ORGA_DK_DELEG_FATO").alias("DOAT_ORGE_DK_DELEG_FATO"),
		col("DOCU_ORGE_ORGA_DK_DELEG_ORIGEM").alias("DOAT_ORGE_DK_ORIGEM"),
		col("DOCU_ORGE_ORGA_DK_VARA").alias("DOAT_ORGE_DK_VARA"),
		col("DOCU_NR_DISTRIBUICAO").alias("DOAT_DOCU_NR_DISTRIBUICAO"),
		col("DOCU_DT_DISTRIBUICAO").alias("DOAT_DOCU_DT_DISTRIBUICAO"),
		col("DOCU_IN_DOCUMENTO_ELETRONICO").alias("DOAT_DOCU_IN_DOC_ELETRONICO"),
		col("DOCU_CLDC_DK").alias("DOAT_CLDC_DK"),
		col("NISI_DS_NIVEL_SIGILO").alias("DOAT_NISI_DS_NIVEL_SIGILO"),
		col("MATE_DESCRICAO").alias("DOAT_MATE_ATRIBUICAO_DOC"),
		col("TPDC_SIGLA").alias("DOAT_TPDC_SIGLA_DOC"),
		col("TPDC_DESCRICAO").alias("DOAT_TPDC_DS_DOCUMENTO"),
		col("DOAT_ORGAO_RESPONSAVEL"),
		col("DOAT_CRAAI_OR"),
		col("DOAT_COMARCA_OR"),
		col("DOAT_FORO_OR"),
		col("DOAT_ORGAO_TP_OR"),
		col("DOAT_ORGAO_A_E_OR"),
		col("DOAT_JUIZO_UNICO_OR"),
		col("DOAT_DT_INICIO_OR"),
		col("DOAT_DT_FIM_OR"),
		col("DOAT_DET_CRIACAO_OR"),
		col("DOAT_ORGAO_CARGA"),
		col("DOAT_CRAAI_CG"),
		col("DOAT_COMARCA_CG"),
		col("DOAT_ORGAO_TP_CG"),
		col("DOAT_ORGAO_A_E_CG"),
		col("DOAT_JUIZO_UNICO_CG"),
		col("DOAT_DT_FIM_CG"),
		col("DOAT_NM_ORGAO_EXTERNO"),
		col("TPOE_DESCRICAO").alias("DOAT_TP_ORGAO_EXTERNO"),
		col("DOAT_NM_DELEF_FATO"),
		col("DOAT_NM_DELEG_ORIGEM"),
		col("DOAT_NM_VARA"),
		col("TPST_DS_TP_SITUACAO").alias("DOAT_TPST_DS_TP_SITUACAO"),
		col("FSDC_DS_FASE").alias("DOAT_FSDC_DS_FASE"),
		col("cod_mgp").alias("DOAT_CD_CLASSE"),
		col("descricao").alias("DOAT_CLASSE"),
		col("hierarquia").alias("DOAT_CLASSE_HIERARQUIA"),
		col("DOAA_DT_ALTERACAO").alias("DOAT_DT_ALTERACAO"),
	]
    documento = spark.table("%s.mcpr_documento" % options["schema_exadata"]).\
		filter("DOCU_DT_CANCELAMENTO IS NULL")
    sigilo = spark.table("%s.mcpr_nivel_sigilo" % options["schema_exadata"])
    materia = spark.table("%s.mprj_materia_mgp" % options["schema_exadata"])
    tipo_doc = spark.table("%s.mcpr_tp_documento" % options["schema_exadata"])
    alteracao = spark.table("%s.mcpr_documento_alteracao" % options["schema_exadata"])
    sit_doc = spark.table("%s.mcpr_tp_situacao_documento" % options["schema_exadata"])
    fase_doc = spark.table("%s.mcpr_fases_documento" % options["schema_exadata"])
    orgao_origem = spark.table("%s.mprj_orgao_ext" % options["schema_exadata"]).select([
        col("ORGE_ORGA_DK").alias("ORG_EXT_ORIGEM_DK"),
		col("ORGE_TPOE_DK").alias("ORG_EXT_TPOE_DK"),
		col("ORGE_NM_ORGAO").alias("DOAT_NM_ORGAO_EXTERNO"),
	])
    orgao_dp_fato = spark.table("%s.mprj_orgao_ext" % options["schema_exadata"]).select([
		col("ORGE_ORGA_DK").alias("ORG_EXT_DP_FATO_DK"),
		col("ORGE_NM_ORGAO").alias("DOAT_NM_DELEF_FATO"),
	])
    orgao_dp_origem = spark.table("%s.mprj_orgao_ext" % options["schema_exadata"]).select([
		col("ORGE_ORGA_DK").alias("ORG_EXT_DP_ORIGEM_DK"),
		col("ORGE_NM_ORGAO").alias("DOAT_NM_DELEG_ORIGEM"),
	])
    orgao_vara = spark.table("%s.mprj_orgao_ext" % options["schema_exadata"]).select([
		col("ORGE_ORGA_DK").alias("ORG_EXT_VARA_DK"),
		col("ORGE_NM_ORGAO").alias("DOAT_NM_VARA"),
	])
    tp_orgao_ext = spark.table("%s.mprj_tp_orgao_ext" % options["schema_exadata"])
    classe_doc = spark.table("%s.mmps_classe_docto" % options["schema_exadata_aux"])
    local_resp = spark.table("%s.orgi_vw_orgao_local_atual" % options["schema_exadata"]).select([
		col("ORLW_DK").alias("LOC_RESP_DK"),
		col("ORLW_ORGI_TPOR_DK").alias("LOC_RESP_TPOR_DK"),
		col("ORLW_ORGI_NM_ORGAO").alias("DOAT_ORGAO_RESPONSAVEL"),
		col("ORLW_REGI_NM_REGIAO").alias("DOAT_CRAAI_OR"),
		col("ORLW_CMRC_NM_COMARCA").alias("DOAT_COMARCA_OR"),
		col("ORLW_COFO_NM_FORO").alias("DOAT_FORO_OR"),
		col("ORLW_ORGI_IN_JUIZO_UNICO").alias("DOAT_JUIZO_UNICO_OR"),
		col("ORLW_ORGI_DT_INICIO").alias("DOAT_DT_INICIO_OR"),
		col("ORLW_ORGI_DT_FIM").alias("DOAT_DT_FIM_OR"),
		col("ORLW_ORGI_DET_CRIACAO").alias("DOAT_DET_CRIACAO_OR"),
	])
    local_carga = spark.table("%s.orgi_vw_orgao_local_atual" % options["schema_exadata"]).select([
		col("ORLW_DK").alias("LOC_CARGA_DK"),
		col("ORLW_ORGI_TPOR_DK").alias("LOC_CARGA_TPOR_DK"),
		col("ORLW_ORGI_NM_ORGAO").alias("DOAT_ORGAO_CARGA"),
		col("ORLW_REGI_NM_REGIAO").alias("DOAT_CRAAI_CG"),
		col("ORLW_CMRC_NM_COMARCA").alias("DOAT_COMARCA_CG"),
		col("ORLW_ORGI_IN_JUIZO_UNICO").alias("DOAT_JUIZO_UNICO_CG"),
		col("ORLW_ORGI_DT_FIM").alias("DOAT_DT_FIM_CG"),
	])
    tp_local_resp = spark.table("%s.orgi_tp_orgao" % options["schema_exadata"]).select([
		col("TPOR_DK").alias("TP_LOC_RESP_DK"),
		col("TPOR_DS_TP_ORGAO").alias("DOAT_ORGAO_TP_OR"),
		col("TPOR_CLASSIFICACAO").alias("DOAT_ORGAO_A_E_OR"),
	])
    tp_local_carga = spark.table("%s.orgi_tp_orgao" % options["schema_exadata"]).select([
		col("TPOR_DK").alias("TP_LOC_CARGA_DK"),
		col("TPOR_DS_TP_ORGAO").alias("DOAT_ORGAO_TP_CG"),
		col("TPOR_CLASSIFICACAO").alias("DOAT_ORGAO_A_E_CG"),
	])

    doc_sigilo = documento.join(sigilo, documento.DOCU_NISI_DK == sigilo.NISI_DK, "left")
    doc_materia = doc_sigilo.join(materia, doc_sigilo.DOCU_MATE_DK == materia.MATE_DK, "left")
    doc_tipo = doc_materia.join(tipo_doc, doc_materia.DOCU_TPDC_DK == tipo_doc.TPDC_DK, "inner")
    # doc_tipo = doc_sigilo.join(tipo_doc, doc_sigilo.DOCU_TPDC_DK == tipo_doc.TPDC_DK, "inner")
    doc_alteracao = doc_tipo.join(alteracao, alteracao.DOAA_DOCU_DK == doc_tipo.DOCU_DK, "inner")
    doc_sit = doc_alteracao.join(sit_doc, doc_alteracao.DOCU_TPST_DK == sit_doc.TPST_DK, "left")
    # doc_sit = doc_tipo.join(sit_doc, doc_tipo.DOCU_TPST_DK == sit_doc.TPST_DK, "left")
    doc_fase = doc_sit.join(fase_doc, doc_sit.DOCU_FSDC_DK == fase_doc.FSDC_DK, "left")
    doc_origem = doc_fase.join(orgao_origem, doc_fase.DOCU_ORGA_DK_ORIGEM == orgao_origem.ORG_EXT_ORIGEM_DK, "left")
    doc_tp_ext = doc_origem.join(tp_orgao_ext, doc_origem.ORG_EXT_TPOE_DK == tp_orgao_ext.TPOE_DK , "left")
    doc_classe = doc_tp_ext.join(classe_doc, doc_tp_ext.DOCU_CLDC_DK == classe_doc.ID , "left")
    # doc_classe = doc_origem.join(classe_doc, doc_origem.DOCU_CLDC_DK == classe_doc.cldc_dk , "left")
    doc_loc_resp = doc_classe.join(local_resp, doc_classe.DOCU_ORGI_ORGA_DK_RESPONSAVEL == local_resp.LOC_RESP_DK , "left")
    doc_tp_loc_resp = doc_loc_resp.join(tp_local_resp, doc_loc_resp.LOC_RESP_TPOR_DK == tp_local_resp.TP_LOC_RESP_DK , "left")
    doc_loc_carga = doc_tp_loc_resp.join(local_carga, doc_tp_loc_resp.DOCU_ORGI_ORGA_DK_CARGA == local_carga.LOC_CARGA_DK , "left")
    doc_tp_carga_resp = doc_loc_carga.join(tp_local_carga, doc_loc_carga.LOC_CARGA_TPOR_DK == tp_local_carga.TP_LOC_CARGA_DK , "left")
    doc_dp_fato = doc_tp_carga_resp.join(orgao_dp_fato, doc_tp_carga_resp.DOCU_ORGE_ORGA_DK_DELEG_FATO == orgao_dp_fato.ORG_EXT_DP_FATO_DK, "left")
    doc_dp_origem = doc_dp_fato.join(orgao_dp_origem, doc_dp_fato.DOCU_ORGE_ORGA_DK_DELEG_ORIGEM == orgao_dp_origem.ORG_EXT_DP_ORIGEM_DK, "left")
    doc_dp_vara = doc_dp_origem.join(orgao_vara, doc_dp_origem.DOCU_ORGE_ORGA_DK_VARA == orgao_vara.ORG_EXT_VARA_DK, "left")

    return doc_dp_vara.select(columns)

def extract_tbau_andamento(spark):
	columns = [
		col("VIST_DOCU_DK").alias("DOAN_DOCU_DK"),
		col("VIST_DK").alias("DOAN_VIST_DK"),
		col("VIST_DT_ABERTURA_VISTA").alias("DOAN_VIST_DT_ABERTURA_VISTA"),
		col("VIST_ORGI_ORGA_DK").alias("DOAN_VIST_ORGI_DK"),
		col("ORLW_ORGI_NM_ORGAO").alias("DOAN_ORGAO_VISTA"),
		col("ORLW_REGI_NM_REGIAO").alias("DOAN_CRAAI_OV"),
		col("ORLW_CMRC_NM_COMARCA").alias("DOAN_COMARCA_OV"),
		col("ORLW_COFO_NM_FORO").alias("DOAN_FORO_OV"),
		col("TPOR_DS_TP_ORGAO").alias("DOAN_ORGAO_TP_OV"),
		col("TPOR_CLASSIFICACAO").alias("DOAN_ORGAO_A_E_OV"),
		col("ORLW_ORGI_IN_JUIZO_UNICO").alias("DOAN_JUIZO_UNICO_OV"),
		col("ORLW_ORGI_DT_INICIO").alias("DOAN_DT_INICIO_OV"),
		col("ORLW_ORGI_DT_FIM").alias("DOAN_DT_FIM_OV"),
		col("ORLW_ORGI_DET_CRIACAO").alias("DOAN_DET_CRIACAO_OV"),
		col("PESS_NM_PESSOA").alias("DOAN_PESS_NM_RESPONSAVEL_ANDAM"),
		col("PCAO_DK").alias("DOAN_PCAO_DK"),
		col("PCAO_DT_ANDAMENTO").alias("DOAN_PCAO_DT_ANDAMENTO"),
		col("STAO_DK").alias("DOAN_STAO_DK_SUB_ANDAMENTO"),
		col("STAO_TPPR_DK").alias("DOAN_TPPR_DK_ANDAMENTO"),
		col("TPPR_TPPR_DK").alias("DOAN_TPPR_TPPR_DK_PAI"),
		col("STAO_IN_RELATADO").alias("DOAN_STAO_IN_RELATADO"),
		col("TPPR_CD_TP_ANDAMENTO").alias("DOAN_TPPR_CD_TP_ANDAMENTO"),
		col("TPPR_DESCRICAO").alias("DOAN_TPPR_DS_ANDAMENTO"),
		col("HIERARQUIA").alias("DOAN_ANDAMENTO_HIERARQUIA"),
		col("TEMPO_ANDAMENTO").alias("DOAN_TEMPO")
	]

	vista = spark.table("%s.mcpr_vista" % options["schema_exadata"])
	pessoa = spark.table("%s.mcpr_pessoa" % options["schema_exadata"])
	vista_resp = vista.join(pessoa, vista.VIST_PESF_PESS_DK_RESP_ANDAM == pessoa.PESS_DK, "left")
	andamento = spark.table("%s.mcpr_andamento" % options["schema_exadata"])
	vista_andam = vista_resp.join(
		andamento,
		[
			vista_resp.VIST_DK == andamento.PCAO_VIST_DK,
			andamento.PCAO_TPSA_DK == 2
		],
		"left"
	).withColumn(
		'TEMPO_ANDAMENTO',
		lit(datediff('PCAO_DT_ANDAMENTO', 'VIST_DT_ABERTURA_VISTA')).cast(IntegerType())
	)
	sub_andamento = spark.table("%s.mcpr_sub_andamento" % options["schema_exadata"])
	vista_suba = vista_andam.join(sub_andamento, vista_andam.PCAO_DK == sub_andamento.STAO_PCAO_DK, "left")
	tipo_andamento = spark.table("%s.mcpr_tp_andamento" % options["schema_exadata"])
	vista_tpand = vista_suba.join(tipo_andamento, vista_suba.STAO_TPPR_DK == tipo_andamento.TPPR_DK, "left")
	hier_andamento = spark.table("%s.mmps_tp_andamento" % options["schema_exadata_aux"])
	vista_hrand = vista_tpand.join(hier_andamento, vista_tpand.TPPR_DK == hier_andamento.ID, "left")
	orgao_local = spark.table("%s.orgi_vw_orgao_local_atual" % options["schema_exadata"])
	vista_orgao = vista_hrand.join(orgao_local, vista_hrand.VIST_ORGI_ORGA_DK == orgao_local.ORLW_DK, "left")
	tp_orgao = spark.table("%s.orgi_tp_orgao" % options["schema_exadata"])
	vista_tp_orgao = vista_orgao.join(tp_orgao, vista_orgao.ORLW_ORGI_TPOR_DK == tp_orgao.TPOR_DK, "left")

	return vista_tp_orgao.select(columns)

def extract_tbau_assunto(spark):
    columns = [
		col("ASDO_DOCU_DK").alias("DASN_DOCU_DK"),
		col("ASDO_DK").alias("DASN_DK"),
		col("ASSU_TX_DISPOSITIVO_LEGAL").alias("DASN_TP_LEGAL"),
		col("ASSU_NM_ASSUNTO").alias("DASN_NM_ASSUNTO"),
		col("ASSU_CD_CNJ").alias("DASN_CD_CNJ"),
		col("ASSU_CD_ASSUNTO").alias("DASN_CD_ASSUNTO"),
		col("ASSU_DK").alias("DASN_ASSU_DK"),
		col("ASSU_ASSU_DK").alias("DASN_ASSU_ASSU_DK_PAI"),
	]

    assunto_documento = spark.table("%s.mcpr_assunto_documento" % options["schema_exadata"])
    assunto = spark.table("%s.mcpr_assunto" % options["schema_exadata"])
    doc_assunto_join = assunto_documento.join(assunto, assunto_documento.ASDO_ASSU_DK == assunto.ASSU_DK, "inner")

    return doc_assunto_join.select(columns).distinct()

def extract_tbau_movimentacao(spark):
	columns = [
		col("ITEM_DOCU_DK").alias("DOMO_DOCU_DK"),
		col("ITEM_DK").alias("DOMO_ITEM_DK"),
		col("MOVI_DK").alias("DOMO_MOVIDK"),
		col("MOVI_ORGA_DK_ORIGEM").alias("DOMO_ORGA_ORIGEM_GUIA"),
		col("ORIG_NM_PESSOA").alias("DOMO_ORGI_ORIGEM_GUIA"),
		col("ORIG_IN_TP_PESSOA").alias("DOMO_TP_ORGI_ORIGEM"),
		col("MOVI_ORGA_DK_DESTINO").alias("DOMO_ORGA_DESTINO_GUIA"),
		col("DEST_NM_PESSOA").alias("DOMO_ORGI_DESTINO_GUIA"),
		col("DEST_IN_TP_PESSOA").alias("DOMO_TP_ORGI_DESTINO"),
		col("MOVI_DT_ENVIO_GUIA").alias("DOMO_MOVI_DT_ENVIO_GUIA"),
		col("MOVI_DT_RECEBIMENTO_GUIA").alias("DOMO_MOVI_DT_RECEBIMENTO_GUIA"),
		col("PCED_DS_PROCEDENCIA").alias("DOMO_DS_PROCEDENCIA_GUIA"),
	]

	movimentacao = spark.table("%s.mcpr_movimentacao" % options["schema_exadata"])
	item = spark.table("%s.mcpr_item_movimentacao" % options["schema_exadata"])
	movi_item = movimentacao.join(item, item.ITEM_MOVI_DK == movimentacao.MOVI_DK, "inner")
	procedencia = spark.table("%s.mcpr_procedencia_documento" % options["schema_exadata"])
	movi_proc = movi_item.join(procedencia, movi_item.MOVI_PCED_DK_PROCEDENCIA == procedencia.PCED_DK, "inner")
	destino = spark.table("%s.mcpr_pessoa" % options["schema_exadata"]).select([
		col("PESS_DK").alias("DEST_DK"),
		col("PESS_NM_PESSOA").alias("DEST_NM_PESSOA"),
		col("PESS_IN_TP_PESSOA").alias("DEST_IN_TP_PESSOA"),
	])
	movi_destino = movi_proc.join(destino, movi_proc.MOVI_ORGA_DK_DESTINO == destino.DEST_DK, "inner")
	origem = spark.table("%s.mcpr_pessoa" % options["schema_exadata"]).select([
		col("PESS_DK").alias("ORIG_DK"),
		col("PESS_NM_PESSOA").alias("ORIG_NM_PESSOA"),
		col("PESS_IN_TP_PESSOA").alias("ORIG_IN_TP_PESSOA"),
	])
	movi_origem = movi_destino.join(origem, movi_destino.MOVI_ORGA_DK_ORIGEM == origem.ORIG_DK, "inner")

	return movi_origem.filter("MOVI_DT_CANCELAMENTO IS NULL").select(columns).distinct()

def extract_tbau_personagem(spark):
	sf1_columns = [
		col("PESF_PESS_DK").alias("PESFDK"),
        col("PESF_SEXO").alias("SEX"),
		col("ESCO_DESCRICAO").alias("ESCOL"),
		col("ECIV_DESCRICAO").alias("ECIVIL"),
		col("CORP_DESCRICAO").alias("CPELE"),
		col("PESF_DT_NASC").alias("DT_NASC"),
	]

	sf2_columns = [
		col("ENPE_PESS_DK").alias("ENPEDK"),
        col("ENDC_CEP").alias("ECEP"),
		col("ECIDA"),
		col("EUFED"),
		col("EBAIR"),
	]

	columns = [
		col("PERS_PESS_DK").alias("DPSG_PERS_PESS_DK"),
        col("PERS_DOCU_DK").alias("DPSG_DOCU_DK"),
		col("PESS_NM_PESSOA").alias("DPSG_PESS_NM_PESSOA"),
		col("TPPE_DESCRICAO").alias("DPSG_TPPE_DESCRICAO"),
		col("TPAT_DS_AUTORIDADE").alias("DPSG_TPAT_DS_AUTORIDADE"),
		col("REND_DESCRICAO").alias("DPSG_REND_DESCRICAO"),
		col("PESS_IN_TP_PESSOA").alias("DPSG_PESS_IN_TP_PESSOA"),
        col("SEX").alias("DPSG_SEXO"),
		col("ESCOL").alias("DPSG_ESCOLARIDADE"),
		col("ECIVIL").alias("DPSG_ESTADO_CIVIL"),
		col("CPELE").alias("DPSG_COR_PELE"),
		col("DT_NASC").alias("DPSG_DT_NASCIMENTO"),
		col("ECEP").alias("DPSG_CEP_PERSONAGEM"),
		col("EBAIR").alias("DPSG_BAIRRO_PERSONAGEM"),
		col("ECIDA").alias("DPSG_CIDADE_PERSONAGEM"),
		col("EUFED").alias("DPSG_UF_PERSONAGEM"),
	]

	pessoa_fisica = spark.table("%s.mcpr_pessoa_fisica" % options["schema_exadata"])
	escolaridade = spark.table("%s.mcpr_escolaridade" % options["schema_exadata"])
	pessoa_escolaridade = pessoa_fisica.join(escolaridade, pessoa_fisica.PESF_ESCO_DK == escolaridade.ESCO_DK, "left")
	estado_civil = spark.table("%s.mcpr_estado_civil" % options["schema_exadata"])
	pessoa_estado_civil = pessoa_escolaridade.join(estado_civil, pessoa_escolaridade.PESF_ECIV_DK == estado_civil.ECIV_DK, "left")
	cor_pele = spark.table("%s.mcpr_cor_pele" % options["schema_exadata"])
	pessoa_cor_pele = pessoa_estado_civil.join(cor_pele, pessoa_estado_civil.PESF_CORP_DK == cor_pele.CORP_DK, "left")
	sf1 = pessoa_cor_pele.select(sf1_columns).distinct()

	end_pes = spark.table("%s.mcpr_endereco_pessoa" % options["schema_exadata"])
	endereco = spark.table("%s.mcpr_enderecos" % options["schema_exadata"]).select(
		["ENDC_DK", "ENDC_BAIR_DK", "ENDC_NM_BAIRRO", "ENDC_CEP", "ENDC_CIDA_DK", "ENDC_NM_CIDADE", "ENDC_NM_ESTADO", "ENDC_UFED_DK"]
	)
	endereco_pessoa = end_pes.join(endereco, end_pes.ENPE_ENDC_DK == endereco.ENDC_DK, "left")
	cidade = spark.table("%s.mprj_cidade" % options["schema_exadata"])
	endereco_cidade = endereco_pessoa.join(cidade, endereco_pessoa.ENDC_CIDA_DK == cidade.CIDA_DK, "left")
	uf = spark.table("%s.mprj_uf" % options["schema_exadata"])
	endereco_uf = endereco_cidade.join(uf, endereco_cidade.CIDA_UFED_DK == uf.UFED_DK, "left")
	bairro = spark.table("%s.mprj_bairro" % options["schema_exadata"])
	endereco_bairro = endereco_uf.join(
		bairro,
		[
			endereco_uf.ENDC_CIDA_DK == bairro.BAIR_CIDA_DK,
			endereco_uf.ENDC_BAIR_DK == bairro.BAIR_DK,
		],
		"left"
	).withColumn(
		'ECIDA',
        coalesce(
			col('CIDA_NM_CIDADE'),
            col('ENDC_NM_CIDADE')
        )
    ).withColumn(
		'EUFED',
        coalesce(
			col('UFED_SIGLA'),
            col('ENDC_NM_ESTADO')
        )
    ).withColumn(
		'EBAIR',
        coalesce(
			col('BAIR_NM_BAIRRO'),
            col('ENDC_NM_BAIRRO')
        )
    )
	sf2 = endereco_bairro.select(sf2_columns).distinct()

	personagem = spark.table("%s.mcpr_personagem" % options["schema_exadata"])
	tipo_personagem = spark.table("%s.mcpr_tp_personagem" % options["schema_exadata"])
	personagem_tipo = personagem.join(tipo_personagem, personagem.PERS_TPPE_DK == tipo_personagem.TPPE_DK, "inner")
	pessoa = spark.table("%s.mcpr_pessoa" % options["schema_exadata"])
	personagem_pessoa = personagem_tipo.join(pessoa, personagem_tipo.PERS_PESS_DK == pessoa.PESS_DK, "inner")
	tipo_autoridade = spark.table("%s.mcpr_tp_autoridade" % options["schema_exadata"])
	personagem_autoridade = personagem_pessoa.join(tipo_autoridade, personagem_pessoa.PERS_TPAT_DK == tipo_autoridade.TPAT_DK, "left")
	perfil = spark.table("%s.mcpr_perfil" % options["schema_exadata"])
	personagem_perfil = personagem_autoridade.join(perfil, personagem_autoridade.PERS_PESF_DK == perfil.PERF_PESF_PESS_DK, "left")
	renda = spark.table("%s.mcpr_faixa_renda" % options["schema_exadata"])
	personagem_renda = personagem_perfil.join(renda, personagem_perfil.PERF_REND_DK == renda.REND_DK, "left")
	personagem_sf1 = personagem_renda.join(
		sf1,
		[
			sf1.PESFDK == personagem_renda.PERS_PESS_DK,
			personagem_renda.PESS_IN_TP_PESSOA.isin(['I', 'J']) == False,
		],
		"left"
	)
	personagem_sf2 = personagem_sf1.join(
		sf2,
		[
			sf2.ENPEDK == personagem_sf1.PERS_PESS_DK,
			personagem_sf1.PESS_IN_TP_PESSOA != 'I',
		],
		"left"
	)

	return personagem_sf2.filter("PERS_DT_FIM IS NULL").select(columns).distinct()

def extract_tbau_consumo(spark):
	columns = [
		col("cd_bem_servico").alias("tmat_cd_bem_servico"),
		col("ds_bem_generico").alias("tmat_ds_bem_generico"),
		col("nm_bem_servico").alias("tmat_nm_bem_servico"),
		col("ds_completa").alias("tmat_ds_completa"),
		col("mes_ano_consumo").alias("tmat_mes_ano_consumo"),
		col("orlw_dk").alias("tmat_orgi_dk"),
		col("orlw_orgi_nm_orgao").alias("tmat_orgi_nm_orgao"),
		col("orlw_cofo_nm_foro").alias("tmat_cofo_nm_foro"),
		col("orlw_cmrc_nm_comarca").alias("tmat_cmrc_nm_comarca"),
		col("orlw_regi_nm_regiao").alias("tmat_regi_nm_regiao"),
		col("sum_atendido").alias("tmat_qt_consumida"),
		col("sg_um").alias("tmat_sg_unidade_medida"),
		col("ds_um").alias("tmat_ds_unidade_medida"),
		col("sum_consumido").alias("tmat_vl_consumido"),
	]
	pre_columns = [
		col("cd_bem_servico"),
		col("ds_bem_generico"),
		col("nm_bem_servico"),
		col("ds_completa"),
		col("qt_atendido"),
		col("sg_um"),
		col("ds_um"),
		col("vl_consumido"),
		col("mes_ano_consumo"),
		col("orlw_dk"),
		col("orlw_orgi_nm_orgao"),
		col("orlw_cofo_nm_foro"),
		col("orlw_cmrc_nm_comarca"),
		col("orlw_regi_nm_regiao"),
	]

	requisicao = spark.table("%s.asin_ax_v_requisicao_consulta" % options["schema_exadata_views"]).\
		filter("CD_SITUACAO_REQ_ATUAL = '003'").\
		filter("QT_ATENDIDO IS NOT NULL").\
		withColumn("vl_consumido", col("vl_atendido")/100).\
		withColumn("mes_ano_consumo", trunc("dt_situacao_req_atual", "month"))
	situacao = spark.table("%s.asin_ax_situacao_req" % options["schema_exadata_views"])
	ua = spark.table("%s.asin_cr_ua" % options["schema_exadata_views"]).select(col("CD_UA").alias("ua_id"))
	servico = spark.table("%s.asin_cr_bem_servico" % options["schema_exadata_views"]).select([
		col("CD_BEM_SERVICO").alias("servico_id"),
		col("CD_BEM_GENERICO"),
		col("nm_bem_servico"),
		col("ds_completa"),
		col("cd_um_elementar"),
	])
	generico = spark.table("%s.asin_cr_bem_generico" % options["schema_exadata_views"]).select([
		col("CD_BEM_GENERICO").alias("generico_id"),
		col("ds_bem_generico"),
	])
	umed = spark.table("%s.asin_cr_um" % options["schema_exadata_views"])
	orgao = spark.table("%s.orgi_vw_orgao_local_atual" % options["schema_exadata"])

	sit_req = requisicao.join(situacao, requisicao.CD_SITUACAO_REQ_ATUAL == situacao.CD_SITUACAO_REQ, "inner")
	sit_ua = sit_req.join(ua, sit_req.CD_UA == ua.ua_id, "inner")
	sit_servico = sit_ua.join(servico, sit_ua.CD_BEM_SERVICO == servico.servico_id, "inner")
	sit_generico = sit_servico.join(generico, sit_servico.CD_BEM_GENERICO == generico.generico_id, "inner")
	sit_umed = sit_generico.join(umed, sit_generico.cd_um_elementar == umed.CD_UM, "inner")
	sit_orgao = sit_umed.join(orgao, sit_umed.CD_UA == orgao.ORLW_ORGI_CDORGAO, "inner").select(pre_columns)
	result = sit_orgao.groupBy(
			"cd_bem_servico", "ds_bem_generico", "nm_bem_servico", "ds_completa", "sg_um", "ds_um", "mes_ano_consumo",
			"orlw_dk", "orlw_orgi_nm_orgao", "orlw_cofo_nm_foro", "orlw_cmrc_nm_comarca", "orlw_regi_nm_regiao"
		).sum("qt_atendido", "vl_consumido").\
		withColumn("sum_atendido", col("sum(qt_atendido)")).\
		withColumn("sum_consumido", col("sum(vl_consumido)"))

	return result.select(columns)

def extract_tbau_endereco(spark):
	columns = [
		col("EDOC_DOCU_DK").alias("DODR_DOCU_DK"),
		concat(
			col("TPLO_DS_LOGRADOURO"),
			lit(" "),
			col("ENDC_LOGRADOURO"),
			lit(" "),
			col("ENDC_NUMERO")
		).alias("DODR_ENDERECO"),
		col("DODR_NM_BAIRRO"),
        col("ENDC_CEP").alias("DODR_CEP"),
        col("DODR_NM_CIDADE"),
        col("DODR_UFED"),
	]

	doc_endereco = spark.table("%s.mcpr_endereco_documento" % options["schema_exadata"])
	endereco = spark.table("%s.mcpr_enderecos" % options["schema_exadata"]).filter("ENDC_CIDA_DK IS NOT NULL")
	cidade = spark.table("%s.mprj_cidade" % options["schema_exadata"])
	estado = spark.table("%s.mprj_uf" % options["schema_exadata"])
	bairro = spark.table("%s.mprj_bairro" % options["schema_exadata"])
	tipo_logradouro = spark.table("%s.mprj_tp_logradouro" % options["schema_exadata"])

	end_documento = doc_endereco.join(endereco, doc_endereco.EDOC_ENDC_DK == endereco.ENDC_DK, "inner")
	end_cidade = end_documento.join(cidade, end_documento.ENDC_CIDA_DK == cidade.CIDA_DK, "left")
	end_estado = end_cidade.join(estado, end_cidade.CIDA_UFED_DK == estado.UFED_DK, "left")
	end_bairro = end_estado.join(bairro, [
		end_estado.ENDC_CIDA_DK == bairro.BAIR_CIDA_DK,
	 	end_estado.ENDC_BAIR_DK == bairro.BAIR_DK,
	], "left")
	end_tp_logradouro = end_bairro.join(tipo_logradouro, end_bairro.ENDC_TPLO_DK == tipo_logradouro.TPLO_DK, "left").\
		withColumn(
            'DODR_NM_BAIRRO',
            coalesce(
                col('BAIR_NM_BAIRRO'),
                col('ENDC_NM_BAIRRO')
            )
        ).\
		withColumn(
            'DODR_NM_CIDADE',
            coalesce(
                col('CIDA_NM_CIDADE'),
                col('ENDC_NM_CIDADE')
            )
        ).\
		withColumn(
            'DODR_UFED',
            coalesce(
                col('UFED_SIGLA'),
                col('ENDC_NM_ESTADO')
            )
        )

	return end_tp_logradouro.select(columns).distinct()

def generate_tbau(spark, generator, schema, table_name):
	dataframe = generator(spark)
	full_table_name = "{}.{}".format(schema, table_name)
	dataframe.coalesce(20).write.format('parquet').saveAsTable(full_table_name, mode='overwrite')

	execute_compute_stats(full_table_name)
	print("{} gravada".format(table_name))

def execute_process(options):

    spark = pyspark.sql.session.SparkSession\
        .builder\
        .appName("tabelas_tbau")\
        .enableHiveSupport()\
        .getOrCreate()

    sc = spark.sparkContext

    schema_exadata_aux = options['schema_exadata_aux']

    generate_tbau(spark, extract_tbau_documento, schema_exadata_aux, "tbau_documento")
    generate_tbau(spark, extract_tbau_andamento, schema_exadata_aux, "tbau_documento_andamento")
    generate_tbau(spark, extract_tbau_assunto, schema_exadata_aux, "tbau_documento_assunto")
    generate_tbau(spark, extract_tbau_movimentacao, schema_exadata_aux, "tbau_documento_movimentacao")
    generate_tbau(spark, extract_tbau_personagem, schema_exadata_aux, "tbau_documento_personagem")
    generate_tbau(spark, extract_tbau_consumo, schema_exadata_aux, "tbau_material_consumo")
    generate_tbau(spark, extract_tbau_endereco, schema_exadata_aux, "tbau_documento_endereco")

if __name__ == "__main__":

    parser = argparse.ArgumentParser(description="Create tables tbau")
    parser.add_argument('-e','--schemaExadata', metavar='schemaExadata', type=str, help='')
    parser.add_argument('-a','--schemaExadataAux', metavar='schemaExadataAux', type=str, help='')
    parser.add_argument('-v','--schemaExadataViews', metavar='schemaExadataViews', type=str, help='')
    parser.add_argument('-i','--impalaHost', metavar='impalaHost', type=str, help='')
    parser.add_argument('-o','--impalaPort', metavar='impalaPort', type=str, help='')
    args = parser.parse_args()

    options = {
        'schema_exadata': args.schemaExadata,
        'schema_exadata_aux': args.schemaExadataAux,
        'schema_exadata_views': args.schemaExadataViews,
        'impala_host' : args.impalaHost,
        'impala_port' : args.impalaPort
    }

    execute_process(options)


import multiprocessing
import time

def pro1(q):
    # q에 데이터를 넣는다
    for i in range(100):
        q.put(str(i))
        time.sleep(0.1)

def pro2(q):
    # q에 데이터를 빼낸다
    for i in range(100):
        item = q.get()
        print(item)
        # JoinableQueue
        q.task_done()

if __name__ == '__main__':
    queue = multiprocessing.JoinableQueue()
    p1 = multiprocessing.Process(target=pro1, args=(queue,))
    p2 = multiprocessing.Process(target=pro2, args=(queue,))

    p1.start()
    p2.start()


from django.conf.urls import url,include
from django.contrib import admin
from django.conf import settings
from django.conf.urls.static import static
from . import views

urlpatterns = [
    url(r"^$", views.HomePage.as_view(), name="home"),
    url(r"^index/$", views.TestPage.as_view(), name="test"),
    url(r"^thanks/$", views.ThanksPage.as_view(), name="thanks"),
    url(r'^admin/', admin.site.urls, name='admin'),
    url(r"^accounts/", include("accounts.urls", namespace="accounts")),
    url(r"^accounts/", include("django.contrib.auth.urls")),
    url(r"^groups/",include("groups.urls", namespace="groups")),
    url(r'^company/', include('company.urls',namespace='company')),
    url(r'^project/', include('project.urls',namespace='project')),
    url(r'^sprint/', include('sprint.urls',namespace='sprint')),
    url(r'^task/', include('task.urls',namespace='task')),
    url(r'^log/', include('log.urls',namespace='log')),
    url(r'^todo/', include('todo.urls',namespace='todo')),
    url(r'^meeting/', include('meeting.urls',namespace='meeting')),

]+ static(settings.MEDIA_URL, document_root=settings.MEDIA_ROOT)


import importlib
from copy import deepcopy
from os import path as osp
from collections import OrderedDict

from pyiqa.utils import get_root_logger, scandir
from pyiqa.utils.registry import ARCH_REGISTRY

from pyiqa.default_model_configs import DEFAULT_CONFIGS

__all__ = ['build_network', 'create_metric']

# automatically scan and import arch modules for registry
# scan all the files under the 'archs' folder and collect files ending with
# '_arch.py'
arch_folder = osp.dirname(osp.abspath(__file__))
arch_filenames = [osp.splitext(osp.basename(v))[0] for v in scandir(arch_folder) if v.endswith('_arch.py')]
# import all the arch modules
_arch_modules = [importlib.import_module(f'pyiqa.archs.{file_name}') for file_name in arch_filenames]

def create_metric(metric_name, eval=True, **opt):
    net_opts = OrderedDict()
    if metric_name in DEFAULT_CONFIGS.keys():
        # load default setting first
        default_opt = DEFAULT_CONFIGS[metric_name]['metric_opts']
        net_opts.update(default_opt)
    # then update with custom setting
    net_opts.update(opt)
    network_type = net_opts.pop('type')
    net = ARCH_REGISTRY.get(network_type)(**net_opts)
    net.lower_better = DEFAULT_CONFIGS[metric_name].get('lower_better', False)
    if eval:
        net.eval()
    logger = get_root_logger()
    logger.info(f'Metric [{net.__class__.__name__}] is created.')
    return net

def build_network(opt):
    opt = deepcopy(opt)
    network_type = opt.pop('type')
    net = ARCH_REGISTRY.get(network_type)(**opt)
    logger = get_root_logger()
    logger.info(f'Network [{net.__class__.__name__}] is created.')
    return net


import json as _json
import re as _re
from typing import List as _List

from mitmproxy.http import HTTPFlow as _HTTPFlow
from mitmproxy.io import FlowReader as _FlowReader
from mitmproxy.io import FlowWriter as _FlowWriter
from mitmproxy.io import tnetstring as _tnetstring

from . import utils as _utils

# ---------- Constants ----------

__ENTITY_PROPERTY_NAMES = [
    'accesstoken',
    'advertisingkey',
    'attackingshipxml',
    'authenticationtype',
    'battleid',
    'defendingshipxml',
    'devicekey',
    'email',
    'emailverificationstatus',
    'facebooktoken',
    'facebooktokenexpirydate',
    'gamecentername',
    'gamecenterfriendcount',
    'googleplayaccesstokenexpirydate',
    'refreshtoken',
    'steamid',
]
__QUERY_PARAM_NAMES = [
    'accesstoken',
    'advertisingkey',
    'battleid',
    'checksum',
    'devicekey',
    'email',
    'facebooktoken',
    'gamecentername',
    'password',
    'refreshtoken',
    'steamid',
    'ticket',
]
__RX_PROPERTIES: _re.Pattern = _re.compile('( (' + '|'.join(__ENTITY_PROPERTY_NAMES) + ')="(.*?)")', _re.IGNORECASE | _re.MULTILINE)

# ---------- Functions ----------

def anonymize_flow(flow: _HTTPFlow) -> _HTTPFlow:
    flow.server_conn.sockname = (None, None)

    for query_param_name, query_param_value in flow.request.query.items():
        if query_param_name.lower() in __QUERY_PARAM_NAMES and query_param_value:
            try:
                int(query_param_value)
                query_param_value = '0' * len(query_param_value)
            except:
                query_param_value = 'x' * len(query_param_value)
        flow.request.query[query_param_name] = query_param_value

    request_content = ''
    if flow.request.content:
        request_content = flow.request.content.decode('utf-8')
        try:
            request_content_dict: dict = _json.loads(request_content)
        except:
            request_content_dict: dict = None
        if request_content_dict:
            # Request Content is a json dictionary
            for query_param_name, query_param_value in request_content_dict.items():
                if query_param_name.lower() in __QUERY_PARAM_NAMES and query_param_value:
                    try:
                        int(query_param_value)
                        query_param_value = '0' * len(query_param_value)
                    except:
                        query_param_value = 'x' * len(query_param_value)
                    request_content_dict[query_param_name] = query_param_value
            request_content = _json.dumps(request_content_dict)
        else:
            # Request Content is most likely a query parameter string
            if '=' in request_content:
                query_params = request_content.split('&')
                request_content_dict = {}
                for query_param in query_params:
                    split_query_param = query_param.split('=')
                    if len(split_query_param) == 2:  # Ignore malformed query parameters or strings that aren't query parameters
                        query_param_name, query_param_value = split_query_param
                        if query_param_name.lower() in __QUERY_PARAM_NAMES and query_param_value:
                            try:
                                int(query_param_value)
                                query_param_value = '0' * len(query_param_value)
                            except:
                                query_param_value = 'x' * len(query_param_value)
                        request_content_dict[query_param_name] = query_param_value
                request_content = '&'.join('='.join((key, value)) for key, value in request_content_dict.items())

        flow.request.content = request_content.encode('utf-8')

    response_content = ''
    if flow.response.content:
        response_content = flow.response.content.decode('utf-8')
        matches = __RX_PROPERTIES.finditer(response_content)
        for match in matches:
            matched_string, property_name, property_value = match.groups()
            try:
                int(property_value)
                property_value = '0' * len(property_value)
            except:
                try:
                    _utils.parse_pss_datetime(property_value)
                    property_value = '2016-01-06T00:00:00'
                except:
                    property_value = 'x' * len(property_value)
            response_content = response_content.replace(matched_string, f' {property_name}="{property_value}"')

        flow.response.content = response_content.encode('utf-8')

    return flow

def anynomize_flows(file_path: str) -> _List[_HTTPFlow]:
    with open(file_path, 'rb') as fp:
        flow_reader: _FlowReader = _FlowReader(fp)
        flows = [anonymize_flow(flow) for flow in flow_reader.stream()]
    return flows

def store_flows(file_path: str, flows: _List[_HTTPFlow]) -> None:
    with open(file_path, 'wb') as fp:
        flow_writer: _FlowWriter = _FlowWriter(fp)
        for flow in flows:
            flow_writer.add(flow)


import datetime
import os
import tensorflow as tf

import numpy as np

class Runner(object):
    def __init__(self, agent, env, train, load_path):
        self.agent = agent
        self.env = env
        self.train = train  # True: entrenar agente, False: se carga agente entrenado

        self.episode = 1
        self.last_10_ep_rewards = []

        self.path = './graphs/' + datetime.datetime.now().strftime("%y%m%d_%H%M") \
                    + ('_train_' if self.train else 'run_') \
                    + type(agent).__name__

        self.writer = tf.summary.create_file_writer(self.path)

        # Se carga modelo entrenado
        if not self.train and load_path is not None and os.path.isdir(load_path):
            self.agent.load_model(load_path)

    def summarize(self):
        """Guarda los valores obtenidos por episodio."""
        '''self.writer.add_summary(tf.Summary(
            value=[tf.Summary.Value(tag='Score per Episode', simple_value=self.score)]),
            self.episode
        )
        if self.train and self.episode % 10 == 0:
            self.agent.save_model(self.path)

            try:
                self.agent.update_target_model() # No se que hace
            except AttributeError:
                ...
        '''
        self.episode += 1

    def run(self, episodes):
        """Entrenamiento del agente en la cantidad de episodios dados."""
        while self.episode <= episodes:
            obs = self.env.reset()
            dist = 0
            new_dist = 0
            self.score = 0
            done = False
            act = 0
            while act < 150:
                if obs.last():
                    break

                state, pos_marine, dist = self.agent.state_marine(obs)

                if obs.first():
                    action = self.agent._SELECT_ARMY
                    act_value = 0

                else:
                    act_value, action = self.agent.step(state, pos_marine)

                obs = self.env.step(action)

                next_state, pos_marine, new_dist = self.agent.state_marine(obs)
                reward = obs.reward
                done = reward > 0
                reward += dist - new_dist

                self.score += reward

                # Guarda las experiencias del agente en cada paso de tiempo.

                self.agent.buffer.add(state, act_value, reward, next_state, done)
                state = next_state
                self.agent.cur_frame += 1

                # Copia los pesos de la red principal hacia la red target.
                if self.agent.cur_frame % self.agent.update_target == 0:
                    self.agent.copy_weights(self.agent.main_nn, self.agent.target_nn)

                # Entrenamiento de la red neuronal.
                if len(self.agent.buffer) > self.agent.batch_size:

                    states, actions, rewards, next_states, dones = self.agent.buffer.sample(self.agent.batch_size)
                    loss = self.agent.train_step(states, actions, rewards, next_states, dones)
                act += 1

            # Decrecimiento del epsilon.
            if self.episode < self.agent.num_episodes:

                self.agent.decrease_epsilon()

            # Guarda recompensas de los ultimos 10 episodios.
            if len(self.last_10_ep_rewards) == 10:
                self.last_10_ep_rewards = self.last_10_ep_rewards[1:]
            self.last_10_ep_rewards.append(self.score)

            # Guarda recompensa y explota el conocimiento del agente cada 10 episodios.
            if self.episode % 10 == 0:
                #aux_reward = agent.explotation(iteraciones)
                mean_rewards = np.mean(self.last_10_ep_rewards)

                print(f'Episode {self.episode}/{self.agent.num_episodes}, Epsilon: {self.agent.epsilon:.3f}, '\
                    f'Reward in last 100 episodes: {mean_rewards:.2f}')

                #episodes.append(episode)
                #eps_history.append(agent.epsilon)
                #prom_rewards_greedy.append(aux_reward)
                #last_100_mean_rewards.append(mean_rewards)

            self.summarize()


#!/usr/bin/env python

from __future__ import print_function

import sys
import argparse

def parse_stdin(in_file):
    d = {}
    for line in in_file:
        if ':' in line:
            split = line.split(':')
            stripped = [s.strip() for s in split]
            #if len(stripped) > 2:
            #  print 'Warning: ignoring remaining column in line \'', line, '\''

            # the split(' ')[0] is mainly to get rid of the ms info in runtime
            d[stripped[0]] = stripped[1].split(' ')[0]

    # Something whent wrong
    if 'reason' in d:
        if 'time' in d['reason']:
            d['runtime'] = 'Timeout'
        else:
            d['runtime'] = 'N/A'
    return d

def print_data(d, keys):
    values = [d[key] for key in keys]

    s = '&'.join(values)
    s += '\\\\'
    print(s)

if __name__ == '__main__':
    argument_parser = argparse.ArgumentParser(
        description=("A program to parse output form the COCP exercises."
                     " Pipe the input from your experiment file into it, "
                     " like so: ./queens | gather_stats.py."))
    # Normally, we would save the results but we're not using them now.
    argument_parser.parse_args()

    d = parse_stdin(sys.stdin)
    print_data(d, keys=['runtime', 'failures'])


"""
    Receive an image

        - Global binarize image
        - Find word (connected component RETR_BOUNDARY)
        - Find Rectilinear Polygon

    Return an list of points in order
"""

import cv2
import numpy as np
import sys
import matplotlib.pyplot as plt

PADDING = 2

def binarize(img):
    gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
    _, bin_img = cv2.threshold(gray, 0, 255, cv2.THRESH_BINARY_INV + cv2.THRESH_OTSU)
    return bin_img

def bbox_words(bin_img):
    contours = cv2.findContours(bin_img, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)[-2]

    list_bboxes = []
    for contour in contours:
        x, y, w, h = cv2.boundingRect(contour)
        list_bboxes.append((x-PADDING, y-PADDING, w+2*PADDING, h+2*PADDING))
    return list_bboxes

def draw_region(img, x1, y1, x2, y2):
    h, w = img.shape[:2]
    x1 = max(0, x1)
    y1 = max(0, y1)
    x2 = min(x2, w)
    y2 = min(y2, h)
    cv2.rectangle(img, (x1, y1), (x2, y2), 255, -1)

def merge_words_2_line(bin_img, list_bboxes):
    list_bboxes.sort(key = lambda bbox: bbox[1])

    cnt = len(list_bboxes)

    for i in range(len(list_bboxes)):
        for j in range(len(list_bboxes)):
            x1, y1, w1, h1 = list_bboxes[i]
            x2, y2, w2, h2 = list_bboxes[j]
            if y1 <= y2 and 2 * (y1 + h1 - y2) >= 0.5 * (h1 + h2):
                x_left, x_right = min(x1, x2), max(x1 + w1, x2 + w2)
                y_left, y_right = min(y1, y2), max(y1 + h1, y2 + h2)
                list_bboxes[i] = x_left, y_left, x_right - x_left, y_right - y_left
                list_bboxes[j] = x_left, y_left, x_right - x_left, y_right - y_left

    highlight_img = bin_img.copy()
    for i in range(cnt):
        x, y, w, h = list_bboxes[i]
        draw_region(highlight_img, x, y, x + w, y + h)

    cv2.imwrite('line.png', highlight_img)
    contours = cv2.findContours(highlight_img, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)[-2]
    return contours

def merge_lines_2_paragraph(bin_img, contours):
    cnt = len(contours)
    highlight_img = bin_img.copy()

    cv2.drawContours(highlight_img, contours, -1, 255, -1)

    # for contour in contours:
    #     x, y, w, h = cv2.boundingRect(contour)
    #     draw_region(highlight_img, x, y, x + w, y + h)

    for contour_1 in contours:
        for contour_2 in contours:
            x1, y1, w1, h1 = cv2.boundingRect(contour_1)
            x2, y2, w2, h2 = cv2.boundingRect(contour_2)
            if (x1 != x2 or y1 != y2) and (y1 < y2 and y1 + h1 + h1//4 >= y2 - h2//4):
                x_left = max(x1, x2)
                x_right = min(x1 + w1, x2 + w2)
                if x_left < x_right:
                    draw_region(highlight_img, x_left, y1 + h1, x_right, y2)
    contours = cv2.findContours(highlight_img, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)[-2]

    if (len(contours) == 0):
        return []
    idx = -1
    for i in range(len(contours)):
        if idx == -1 or cv2.contourArea(contours[idx]) < cv2.contourArea(contours[i]):
            idx = i
    epsilon = 0.001*cv2.arcLength(contours[idx], True)
    approx = cv2.approxPolyDP(contours[idx], epsilon, True)

    highlight_img = bin_img.copy()
    for i in range(len(approx)):
        j = (i + 1) % len(approx)
        cv2.line(highlight_img, (approx[i][0][0], approx[i][0][1]), (approx[j][0][0], approx[j][0][1]), 255)
    cv2.imwrite('highlight.png', highlight_img)

    return [(approx[i][0][0], approx[i][0][1]) for i in range(len(approx))]

def main(img):
    cv2.imwrite('crop.png', img)
    bin_img = binarize(img)
    list_bboxes = bbox_words(bin_img)
    line_contours = merge_words_2_line(bin_img, list_bboxes)
    return merge_lines_2_paragraph(bin_img, line_contours)

if __name__ == '__main__':
    img_path = sys.argv[1]
    img = cv2.imread(img_path)
    print(main(img))


import logging
def get_logger():
    """Get logging."""
    logging.getLogger('matplotlib.font_manager').setLevel(logging.WARNING)
    logger = logging.getLogger()
    logger.setLevel(logging.DEBUG)
    formatter = logging.Formatter(
        '%(asctime)s - %(name)s - %(levelname)s: - %(message)s',
        datefmt='%Y-%m-%d %H:%M:%S')
    ch = logging.StreamHandler()
    ch.setLevel(logging.DEBUG)
    ch.setFormatter(formatter)
    logger.addHandler(ch)
    return logger


import discord
import bdg
import enum
import requests
import bs4
import datetime

class BrawlModes(enum.Enum):
	BRAWLBALL    = "brawlBall"
	SOLOSHOWDOWN = "soloShowdown"
	DUOSHOWDOWN  = "duoShowdown"
	GEMGRAB      = "gemGrab"
	BOUNTY       = "bounty"
	HOTZONE      = "hotZone"
	KNOCKOUT     = "knockout"
	HEIST        = "heist"
	SIEGE        = "siege"

names = {
	BrawlModes.BRAWLBALL: "FUTE-BRAWL",
	BrawlModes.SOLOSHOWDOWN: "COMBATE SOLO",
	BrawlModes.DUOSHOWDOWN: "COMBATE DUPLO",
	BrawlModes.GEMGRAB: "PIQUE-GEMA",
	BrawlModes.BOUNTY: "CAÇA ESTRELAS",
	BrawlModes.HOTZONE: "ZONA ESTRATÉGICA",
	BrawlModes.KNOCKOUT: "NOCAUTE",
	BrawlModes.HEIST: "ROUBO",
	BrawlModes.SIEGE: "ENCURRALADO"
}

colors = {
	BrawlModes.BRAWLBALL: 0x8CA0DF,
	BrawlModes.SOLOSHOWDOWN: 0x81D621,
	BrawlModes.DUOSHOWDOWN: 0x81D621,
	BrawlModes.GEMGRAB: 0x9B3DF3,
	BrawlModes.BOUNTY: 0x01CFFF,
	BrawlModes.HOTZONE: 0xE33C50,
	BrawlModes.KNOCKOUT: 0xF7831C,
	BrawlModes.HEIST: 0xD65CD3,
	BrawlModes.SIEGE: 0xF04F32
}

class BrawlMetasCommand(discord.app_commands.Command):

	def __init__(self, bot: bdg.BotDusGuri):
		self.bot = bot
		super().__init__(
			name="brawl_meta",
			description="Mostra uma lista dos brawlers meta de cada modo de jogo",
			callback=self.on_command
		)

	async def on_command(self, i: discord.Interaction, modo: BrawlModes, top: int = 10):

		await i.response.defer(thinking=True)

		data = self.get_metas(modo, top)
		desc = str()

		for brawler in data['brawlers']:
			desc += "{rank}. **{name}** - `{star}%`\n".format(**brawler)

		embed = discord.Embed(
			title=f"Top {top} - {names[modo]}",
			color=colors[modo],
			description=desc
		)

		# Definindo o autor como o Brawl Ace
		embed.set_author(
			name="Brawl Ace",
			url="https://brawlace.com/meta",
			icon_url="https://brawlace.com/assets/images/icon.png?v=22.87"
		)

		# Pegando o ícone do modo de jogo e definindo-o como a thumbnail
		embed.set_thumbnail(url=data['icon'])

		# Definindo legenda
		embed.set_footer(text="(%) Porcentagem de Craque")

		# definindo como timestamp como tempo da ultima atualização
		embed.timestamp = data['last_update']

		await i.followup.send(embed=embed)

	def get_metas(self, mode: BrawlModes, count: int = 10) -> dict[str, any]:

		html = bs4.BeautifulSoup(requests.get("https://brawlace.com/meta", cookies={"lang": "pt"}).content, "html.parser")
		div = html.select_one(f"#gameModeData{mode.value}")
		data = {}

		# Pegando icone do modo de jogo
		data['icon'] = div.select_one("h3 img")['src']

		# Pegando o tempo da ultima atualização
		last_update = html.select_one("div.input-group option", selected=True).text
		last_update = datetime.datetime.strptime(last_update, "%Y-%m-%d %H:%M:%S")
		last_update -= datetime.timedelta(hours=7) # Convertendo de UTC+7 para UTC+0

		data['last_update'] = last_update

		# Definindo brawlers
		data['brawlers'] = []

		brawlers = div.select_one(f"table tbody").find_all("tr")

		count = sorted((3, count, len(brawlers)))[1] # Mesma coisa que .clamp()
		for i in range(count):
			info = {}

			brawler = brawlers[i].find_all('td')
			info['rank'] = i + 1
			info['name'] = brawler[1].text
			info['star'] = float(brawler[3].text[0:-2]) # Formatando de '3.33 %' para 3.33

			data['brawlers'].append(info)

		return data


# -*- coding: utf-8 -*-

from threading import Thread
import queue
import json
import os , sys
#导入requests库(请求和页面抓取)
import requests
#导入time库(设置抓取Sleep时间)
import time
#导入random库(生成乱序随机数)
import random
#导入正则库(从页面代码中提取信息)
import re
#导入数值计算库(常规计算)
import numpy as np
from PIL import Image
from wordcloud import WordCloud
#导入科学计算库(拼表及各种分析汇总)
import pandas as pd
#导入绘制图表库(数据可视化)
import matplotlib.pyplot as plt
#导入结巴分词库(分词)
import jieba as jb
#导入结巴分词(关键词提取)
import jieba.analyse

f=open("rest_idx.txt", 'r')

#设置请求中头文件的信息
headers = {'User-Agent':'Mozilla/5.0 '
                        '(Windows NT 10.0; Win64; x64) '
                        'AppleWebKit/537.36 (KHTML,'
                        ' like Gecko) Chrome/76.0.3809.132 '
                        'Safari/537.36',
#'Accept':'text/html;q=0.9,*/*;q=0.8',
#'Accept-Charset':'ISO-8859-1,utf-8;q=0.7,*;q=0.3',
#"Accept-Language": "zh-CN,zh;q=0.9",
#'Connection':'close',
'Referer':'https://www.jd.com/'
}

q = queue.Queue()
NUM = 5
JOBS = 10
#cookies=***
def crawl_jd_cmt_tag(prdtId):# change for url

    url=r"https://sclub.jd.com/comment/" \
        r"productPageComments.action?callback=fetchJSON_comment98vv106813&" \
        r"productId={}&" \
        r"score=0&sortType=5&page=0&pageSize=10&isShadowSku=0&fold=1".format(prdtId)
    comment_tag_path = r'C:\TC-prog\JetBrain_pycharm_TC' \
                       r'\PycharmProjects\Crawler_EEFinal' \
                       r'\jd_cmt_tags\httpsitem.jd.com{}.html.txt'.format(prdtId)

    try:
        r=requests.get(url,headers=headers,timeout=5)
        r.raise_for_status()
    except:
        print ('爬取失败')

    raw = r.text[:]
    pos = 0
    for i in range(len(raw)):
        if raw[i] == '(':
            pos = i
    try:
        # data0 = re.sub(u'^fetchJSON_comment98vv106813\(', '', html)
        r_json_str = r.text[pos + 1:-2]
        # print (r_json_str)
        r_json_obj=json.loads(r_json_str,strict=False)
        print (r_json_obj)
        r_json_tags=r_json_obj['hotCommentTagStatistics']
        print ('狗东评论标签：')
        # 追加模式，逐行写入

        for r_json_tag in r_json_tags:
            with open(comment_tag_path,'a+') as file:
                file.write(r_json_tag['name']+
                           '\t'+str(r_json_tag['count'])+'\n')
                print(r_json_tag['name']+
                      '\t'+str(r_json_tag['count']))
    except:
        print('failed')

def run():
    global f
    for line in f.readlines():
        try:
            pp = line.split('\t')
            webpage = pp[1].strip('\n')
            print(webpage)
            temp = webpage[14:-5]
            pos = 0
            for i in range(len(temp)):
                if temp[i] == 'm':
                    pos = i
            itemID = temp[pos + 1:]

            if (len(webpage) > 35):
                continue

            crawl_jd_cmt_tag(itemID)
            time.sleep(random.random() * 3)
        except:
            print("Invalid Input!")

def working():
    while True:
        #arguments = q.get()
        run()
        q.task_done()

#fork NUM个线程等待队列
for i in range(NUM):
    t = Thread(target=working)
    t.setDaemon(True)
    t.start()
#把JOBS排入队列
for i in range(JOBS):
    q.put(i)
#阻塞，等待所有JOBS完成
q.join()
f.close()


# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.db import models, migrations
import django.utils.timezone
import django_extensions.db.fields

class Migration(migrations.Migration):

    dependencies = [
    ]

    operations = [
        migrations.CreateModel(
            name='Station',
            fields=[
                ('id', models.AutoField(primary_key=True, serialize=False, auto_created=True, verbose_name='ID')),
                ('created', django_extensions.db.fields.CreationDateTimeField(blank=True, verbose_name='created', editable=False, default=django.utils.timezone.now)),
                ('modified', django_extensions.db.fields.ModificationDateTimeField(blank=True, verbose_name='modified', editable=False, default=django.utils.timezone.now)),
                ('name', models.TextField()),
            ],
            options={
                'ordering': ('-modified', '-created'),
                'get_latest_by': 'modified',
                'abstract': False,
            },
        ),
    ]


# -*- coding:utf-8 -*-
from openpyxl import Workbook
from datetime import datetime

def write_excel(filename):
    wb = Workbook()  # load_work(filename)
    ws = wb.create_sheet('sheet', 0)
    wb.remove('sheet')

    ws = wb.active  # default Sheet
    ws.title = 'Pi'
    ws['A1'] = 3.1415926
    ws['A2'] = datetime.now().strftime('%Y-%m-%d')
    ws['A3'] = '=sum(A1:A5)'

    row = [1, 2, 3, 4]
    ws.append(row)

    rows = [
        ['id', 'name', 'department'],
        ['001', 'lee', 'cs'],
        ['002', 'lee', 'ma']
    ]
    ws.append(rows)

    wb.save(filename)