import json
import lxml
import mock
import StringIO
import tempfile
import unittest

from swift.common import swob

from s3_sync import shunt
from s3_sync import sync_s3
from s3_sync import sync_swift
from s3_sync import utils

class FakeSwift(object):
    def __init__(self):
        self.calls = []

    def __call__(self, env, start_response):
        self.calls.append(env)
        # Let the tests set up the responses they want
        if env.get('__test__.response_dict'):
            resp_dict = env['__test__.response_dict']
            method = env['REQUEST_METHOD']
            if method in resp_dict:
                status = resp_dict[method].get('status', '200 OK')
                headers = resp_dict[method].get('headers', [])
                start_response(status, headers)
                return resp_dict[method].get('body', '')

        status = env.get('__test__.status', '200 OK')
        headers = env.get('__test__.headers', [])
        body = env.get('__test__.body', ['pass'])

        start_response(status, headers)
        return body

class TestShunt(unittest.TestCase):
    def setUp(self):
        self.patchers = [mock.patch(name) for name in (
            's3_sync.sync_swift.SyncSwift.shunt_object',
            's3_sync.sync_s3.SyncS3.shunt_object',
            's3_sync.sync_swift.SyncSwift.list_objects',
            's3_sync.sync_s3.SyncS3.list_objects')]
        self.mock_shunt_swift = self.patchers[0].__enter__()
        self.mock_shunt_swift.return_value = (
            200, [
                ('Remote-x-openstack-request-id', 'also some trans id'),
                ('Remote-x-trans-id', 'some trans id'),
                ('CONNECTION', 'bad'),
                ('keep-alive', 'bad'),
                ('proxy-authenticate', 'bad'),
                ('proxy-authorization', 'bad'),
                ('te', 'bad'),
                ('trailer', 'bad'),
                ('Transfer-Encoding', 'bad'),
                ('Upgrade', 'bad'),
                ('Content-Length', len('remote swift'))
            ], StringIO.StringIO('remote swift'))
        self.mock_shunt_s3 = self.patchers[1].__enter__()
        self.mock_shunt_s3.return_value = (
            200, [
                ('Remote-x-amz-id-2', 'also some trans id'),
                ('Remote-x-amz-request-id', 'some trans id'),
                ('CONNECTION', 'bad'),
                ('keep-alive', 'bad'),
                ('proxy-authenticate', 'bad'),
                ('proxy-authorization', 'bad'),
                ('te', 'bad'),
                ('trailer', 'bad'),
                ('Transfer-Encoding', 'bad'),
                ('Upgrade', 'bad')], ['remote s3'])

        self.mock_list_swift = self.patchers[2].__enter__()
        self.mock_list_s3 = self.patchers[3].__enter__()

        self.conf = {
            'containers': [
                {
                    'account': 'AUTH_a',
                    'container': u'sw\u00e9ft',
                    'propagate_delete': False,
                    'protocol': 'swift',
                    'aws_bucket': 'dest-container',
                    'aws_identity': 'user',
                    'aws_secret': 'key',
                    'aws_endpoint': 'https://swift.example.com/auth/v1.0',
                    'restore_object': True,
                },
                {
                    'account': 'AUTH_a',
                    'container': 's3',
                    'propagate_delete': False,
                    'aws_bucket': 'dest-bucket',
                    'aws_identity': 'user',
                    'aws_secret': 'key',
                },
                {
                    'account': 'AUTH_b',
                    'container': '/*',
                    'propagate_delete': False,
                    'aws_bucket': 'dest-bucket',
                    'aws_identity': 'user',
                    'aws_secret': 'key',
                },
                {
                    'account': 'AUTH_tee',
                    'container': 'tee',
                    'propagate_delete': False,
                    'restore_object': True,
                    'aws_bucket': 'dest-bucket',
                    'aws_identity': 'user',
                    'aws_secret': 'key',
                }]
        }

        with tempfile.NamedTemporaryFile() as fp:
            json.dump(self.conf, fp)
            fp.flush()
            self.app = shunt.filter_factory(
                {'conf_file': fp.name})(FakeSwift())

    def tearDown(self):
        for patcher in self.patchers:
            patcher.__exit__()

    def test_bad_config_noops(self):
        app = shunt.filter_factory(
            {'conf_file': '/etc/doesnt/exist'})(FakeSwift())
        self.assertEqual(app.sync_profiles, {})

        with tempfile.NamedTemporaryFile() as fp:
            # empty
            app = shunt.filter_factory(
                {'conf_file': fp.name})(FakeSwift())
            self.assertEqual(app.sync_profiles, {})

            # not json
            fp.write('{"containers":')
            fp.flush()
            app = shunt.filter_factory(
                {'conf_file': fp.name})(FakeSwift())
            self.assertEqual(app.sync_profiles, {})

    def test_init(self):
        self.maxDiff = None
        self.assertEqual(self.app.sync_profiles, {
            ('AUTH_a', 'sw\xc3\xa9ft'): {
                'account': 'AUTH_a',
                'container': 'sw\xc3\xa9ft'.decode('utf-8'),
                'propagate_delete': False,
                'protocol': 'swift',
                'aws_bucket': 'dest-container',
                'aws_identity': 'user',
                'aws_secret': 'key',
                'aws_endpoint': 'https://swift.example.com/auth/v1.0',
                'restore_object': True,
            },
            ('AUTH_a', 's3'): {
                'account': 'AUTH_a',
                'container': 's3',
                'propagate_delete': False,
                'aws_bucket': 'dest-bucket',
                'aws_identity': 'user',
                'aws_secret': 'key',
            },
            ('AUTH_b', '/*'): {
                'account': 'AUTH_b',
                'container': '/*',
                'propagate_delete': False,
                'aws_bucket': 'dest-bucket',
                'aws_identity': 'user',
                'aws_secret': 'key',
            },
            ('AUTH_tee', 'tee'): {
                'account': 'AUTH_tee',
                'container': 'tee',
                'propagate_delete': False,
                'restore_object': True,
                'aws_bucket': 'dest-bucket',
                'aws_identity': 'user',
                'aws_secret': 'key',
            },
        })

    def test_unshunted_requests(self):
        def _do_test(path, method='GET'):
            req = swob.Request.blank(path, method=method, environ={
                'swift.trans_id': 'local trans id'})
            req.call_application(self.app)
            self.assertEqual(self.mock_shunt_swift.mock_calls, [])
            self.assertEqual(self.mock_shunt_s3.mock_calls, [])

        def _test_methods(path, methods=('OPTIONS', 'GET', 'HEAD', 'PUT',
                                         'POST', 'DELETE', 'COPY')):
            if 'OPTIONS' in methods:
                _do_test(path, 'OPTIONS')
            if 'GET' in methods:
                _do_test(path, 'GET')
            if 'HEAD' in methods:
                _do_test(path, 'HEAD')
            if 'PUT' in methods:
                _do_test(path, 'PUT')
            if 'POST' in methods:
                _do_test(path, 'POST')
            if 'DELETE' in methods:
                _do_test(path, 'DELETE')
            # doesn't necessarily apply to all paths, but whatever
            if 'COPY' in methods:
                _do_test(path, 'COPY')

        # Only shunt object GETs
        _test_methods('/some/weird/non/swift/path')
        _test_methods('/v1/AUTH_a')
        _test_methods('/v1/AUTH_a/')
        # Not an affected container
        _test_methods('/v1/AUTH_a/c')
        _test_methods('/v1/AUTH_a/c/')
        _test_methods('/v1/AUTH_a/c/o')
        # Affected container, but not relevant methods
        _test_methods(u'/v1/AUTH_a/sw\u00e9ft',
                      ('OPTIONS', 'HEAD', 'PUT', 'POST', 'DELETE'))
        _test_methods(u'/v1/AUTH_a/sw\u00e9ft/',
                      ('OPTIONS', 'HEAD', 'PUT', 'POST', 'DELETE'))
        _test_methods('/v1/AUTH_a/s3',
                      ('OPTIONS', 'HEAD', 'PUT', 'POST', 'DELETE'))
        _test_methods('/v1/AUTH_a/s3/',
                      ('OPTIONS', 'HEAD', 'PUT', 'POST', 'DELETE'))
        _test_methods(u'/v1/AUTH_a/sw\u00e9ft/o',
                      ('OPTIONS', 'PUT', 'POST', 'DELETE'))
        _test_methods('/v1/AUTH_a/s3/o',
                      ('OPTIONS', 'PUT', 'POST', 'DELETE'))

    @mock.patch.object(sync_swift.SyncSwift, 'get_manifest')
    @mock.patch.object(sync_s3.SyncS3, 'get_manifest')
    def test_object_shunt(self, mock_s3_manifest, mock_swift_manifest):
        def _test_no_shunt(path, status):
            req = swob.Request.blank(path, environ={
                '__test__.status': status,
                'swift.trans_id': 'local trans id'})
            req.call_application(self.app)
            self.assertEqual(self.mock_shunt_swift.mock_calls, [])
            self.assertEqual(self.mock_shunt_s3.mock_calls, [])
        # Not an affected container
        _test_no_shunt('/v1/AUTH_a/c/o', '404 Not Found')
        _test_no_shunt('/v1/AUTH_a/c/o', '404 Not Found')
        # Affected container, but not 404
        _test_no_shunt(u'/v1/AUTH_a/sw\u00e9ft/o', '200 OK')
        _test_no_shunt('/v1/AUTH_a/s3/o', '200 OK')
        _test_no_shunt(u'/v1/AUTH_a/sw\u00e9ft/o', '503 Not Available')
        _test_no_shunt('/v1/AUTH_a/s3/o', '400 Bad Request')

        # Do the shunt!
        def _test_shunted(path, expect_s3):
            req = swob.Request.blank(path, environ={
                '__test__.status': '404 Not Found',
                'swift.trans_id': 'local trans id'})
            status, headers, body_iter = req.call_application(self.app)
            if expect_s3:
                self.assertEqual(
                    self.mock_shunt_swift.mock_calls, [])
                self.assertEqual(self.mock_shunt_s3.mock_calls, [
                    mock.call(mock.ANY, path.split('/', 4)[4])])
                received_req = self.mock_shunt_s3\
                    .mock_calls[0][1][0]
                self.assertEqual(req.environ, received_req.environ)
                self.assertEqual(status, '200 OK')
                self.assertEqual(headers, [
                    ('Remote-x-amz-id-2', 'also some trans id'),
                    ('Remote-x-amz-request-id', 'some trans id'),
                ])
                self.assertEqual(b''.join(body_iter), b'remote s3')
                self.mock_shunt_s3.reset_mock()
            else:
                self.assertEqual(self.mock_shunt_s3.mock_calls, [])
                self.assertEqual(self.mock_shunt_swift.mock_calls, [
                    mock.call(mock.ANY, path.split('/', 4)[4])])
                received_req = self.mock_shunt_swift\
                    .mock_calls[0][1][0]
                self.assertEqual(req.environ, received_req.environ)
                self.assertEqual(status, '200 OK')
                self.assertEqual(headers, [
                    ('Remote-x-openstack-request-id', 'also some trans id'),
                    ('Remote-x-trans-id', 'some trans id'),
                    ('Content-Length', 12)
                ])
                self.assertEqual(b''.join(body_iter), b'remote swift')
                self.mock_shunt_swift.reset_mock()
        _test_shunted(u'/v1/AUTH_a/sw\u00e9ft/o', False)
        _test_shunted('/v1/AUTH_a/s3/o', True)
        _test_shunted('/v1/AUTH_b/c1/o', True)
        _test_shunted('/v1/AUTH_b/c2/o', True)

    @mock.patch.object(sync_swift.SyncSwift, 'get_manifest')
    @mock.patch.object(sync_s3.SyncS3, 'get_manifest')
    @mock.patch.object(sync_swift.SyncSwift, 'shunt_object')
    @mock.patch.object(sync_s3.SyncS3, 'shunt_object')
    def test_tee(self, mock_s3_shunt, mock_swift_shunt, mock_s3_get_manifest,
                 mock_swift_get_manifest):
        payload = 'bytes from remote'
        responses = [
            ('AUTH_tee/tee',
             (200,
              [('Content-Length', len(payload)),
               (utils.SLO_HEADER, 'True'),
               ('etag', 'deadbeef-2')],
              StringIO.StringIO(payload)),
             mock_s3_shunt, True),
            (u'AUTH_a/sw\u00e9ft',
             (200,
              [('Content-Length', len(payload)),
               (utils.SLO_HEADER, 'True')],
              StringIO.StringIO(payload)),
             mock_swift_shunt, True),
            ('AUTH_tee/tee',
             (200, [('Content-Length', len(payload))],
              StringIO.StringIO(payload)),
             mock_s3_shunt, True),
            (u'AUTH_a/sw\u00e9ft',
             (200, [('Content-Length', len(payload))],
              StringIO.StringIO(payload)),
             mock_swift_shunt, True)
        ]

        env = {
            '__test__.response_dict': {
                'GET': {
                    'status': '404 Not Found'
                }
            }
        }

        for path, resp, mock_call, is_put_back in responses:
            if dict(resp[1]).get('etag', '').endswith('-2') or\
                    utils.SLO_HEADER in dict(resp[1]):
                is_slo = True
            else:
                is_slo = False
            if is_slo:
                manifest = [{
                    'bytes': len(payload),
                    'name': '/segments/part1'}]
                mock_s3_get_manifest.return_value = manifest
                mock_swift_get_manifest.return_value = manifest
            mock_call.return_value = resp
            req = swob.Request.blank(u'/v1/%s/foo' % path, environ=env)
            status, headers, body_iter = req.call_application(self.app)
            resp_body = ''
            while True:
                data = body_iter.read()
                if not data:
                    break
                resp_body += data
            if not is_put_back:
                self.assertEqual(1, len(self.app.app.calls))
            elif is_slo:
                self.assertEqual(4, len(self.app.app.calls))
                self.assertEqual(
                    'PUT', self.app.app.calls[1]['REQUEST_METHOD'])
                account = path.split('/', 1)[0]
                self.assertEqual((u'/v1/%s/segments' % account),
                                 self.app.app.calls[1]['PATH_INFO'])
                self.assertEqual(
                    'PUT', self.app.app.calls[2]['REQUEST_METHOD'])
                self.assertEqual((u'/v1/%s/segments/part1' % account),
                                 self.app.app.calls[2]['PATH_INFO'])
                self.assertEqual(
                    'PUT', self.app.app.calls[3]['REQUEST_METHOD'])
                self.assertEqual((u'/v1/%s/foo' % path).encode('utf-8'),
                                 self.app.app.calls[3]['PATH_INFO'])
                self.assertEqual('multipart-manifest=put',
                                 self.app.app.calls[3]['QUERY_STRING'])
            else:
                self.assertEqual(2, len(self.app.app.calls))
                self.assertEqual(
                    'PUT', self.app.app.calls[1]['REQUEST_METHOD'])
                self.assertEqual((u'/v1/%s/foo' % path).encode('utf-8'),
                                 self.app.app.calls[1]['PATH_INFO'])
            self.assertEqual('GET', self.app.app.calls[0]['REQUEST_METHOD'])
            self.assertEqual((u'/v1/%s/foo' % path).encode('utf-8'),
                             self.app.app.calls[0]['PATH_INFO'])
            self.assertEqual(payload, resp_body)
            mock_call.reset_mock()
            self.app.app.calls = []

    def test_list_container_no_shunt(self):
        req = swob.Request.blank(
            '/v1/AUTH_a/foo',
            environ={'__test__.status': '200 OK',
                     'swift.trans_id': 'id'})
        req.call_application(self.app)

        self.assertEqual(self.mock_list_swift.mock_calls, [])
        self.assertEqual(self.mock_list_s3.mock_calls, [])

    def test_list_container_shunt_s3(self):
        self.mock_list_s3.side_effect = [
            (200, [{'name': 'abc',
                    'hash': 'ffff',
                    'bytes': 42,
                    'last_modified': 'date',
                    'content_type': 'type'},
                   {'name': u'unicod\xe9',
                    'hash': 'ffff',
                    'bytes': 1000,
                    'last_modified': 'date',
                    'content_type': 'type'}]),
            (200, [])]
        req = swob.Request.blank(
            '/v1/AUTH_a/s3',
            environ={'__test__.status': '200 OK',
                     '__test__.body': '[]',
                     'swift.trans_id': 'id'})
        status, headers, body_iter = req.call_application(self.app)
        self.assertEqual(self.mock_shunt_swift.mock_calls, [])
        self.mock_list_s3.assert_has_calls([
            mock.call('', 10000, '', ''),
            mock.call('unicod\xc3\xa9', 10000, '', '')])
        names = body_iter.split('\n')
        self.assertEqual(['abc', u'unicod\xe9'], names)

    def test_list_container_shunt_s3_xml(self):
        elements = [{'name': 'abc',
                     'hash': 'ffff',
                     'bytes': 42,
                     'last_modified': 'date',
                     'content_type': 'type'},
                    {'name': u'unicod\xc3\xa9',
                     'hash': 'ffff',
                     'bytes': 1000,
                     'last_modified': 'date',
                     'content_type': 'type'}]
        self.mock_list_s3.side_effect = [
            (200, elements), (200, [])]
        req = swob.Request.blank(
            '/v1/AUTH_a/s3?format=xml',
            environ={'__test__.status': '200 OK',
                     '__test__.body': '[]',
                     'swift.trans_id': 'id'})
        status, headers, body_iter = req.call_application(self.app)
        self.assertEqual(self.mock_shunt_swift.mock_calls, [])
        self.mock_list_s3.assert_has_calls([
            mock.call('', 10000, '', ''),
            mock.call(u'unicod\xc3\xa9'.encode('utf-8'), 10000, '', '')])
        root = lxml.etree.fromstring(body_iter)
        context = lxml.etree.iterwalk(root, events=("start", "end"))
        element_index = 0
        cur_elem_properties = {}
        for action, elem in context:
            if action == 'end':
                if elem.tag == 'container':
                    self.assertEqual('s3', elem.get('name'))
                elif elem.tag == 'object':
                    self.assertEqual(elements[element_index],
                                     cur_elem_properties)
                    element_index += 1
                else:
                    try:
                        int_value = int(elem.text)
                        cur_elem_properties[elem.tag] = int_value
                    except ValueError:
                        cur_elem_properties[elem.tag] = elem.text

    def test_list_container_accept_xml(self):
        elements = [{'name': 'abc',
                     'hash': 'ffff',
                     'bytes': 42,
                     'last_modified': 'date',
                     'content_type': 'type'},
                    {'name': u'unicod\xc3\xa9',
                     'hash': 'ffff',
                     'bytes': 1000,
                     'last_modified': 'date',
                     'content_type': 'type'}]
        self.mock_list_s3.side_effect = [
            (200, elements), (200, [])]
        req = swob.Request.blank(
            '/v1/AUTH_a/s3',
            environ={'__test__.status': '200 OK',
                     '__test__.body': '[]',
                     'swift.trans_id': 'id'},
            headers={'Accept': 'application/xml'})
        status, headers, body_iter = req.call_application(self.app)
        self.assertEqual(self.mock_shunt_swift.mock_calls, [])
        self.mock_list_s3.assert_has_calls([
            mock.call('', 10000, '', ''),
            mock.call(u'unicod\xc3\xa9'.encode('utf-8'), 10000, '', '')])
        root = lxml.etree.fromstring(body_iter)
        context = lxml.etree.iterwalk(root, events=("start", "end"))
        element_index = 0
        cur_elem_properties = {}
        for action, elem in context:
            if action == 'end':
                if elem.tag == 'container':
                    self.assertEqual('s3', elem.get('name'))
                elif elem.tag == 'object':
                    self.assertEqual(elements[element_index],
                                     cur_elem_properties)
                    element_index += 1
                else:
                    try:
                        int_value = int(elem.text)
                        cur_elem_properties[elem.tag] = int_value
                    except ValueError:
                        cur_elem_properties[elem.tag] = elem.text

    def test_list_container_shunt_s3_json(self):
        elements = [{'name': 'abc',
                     'hash': 'ffff',
                     'bytes': 42,
                     'last_modified': 'date',
                     'content_type': 'type'},
                    {'name': u'unicod\xc3\xa9',
                     'hash': 'ffff',
                     'bytes': 1000,
                     'last_modified': 'date',
                     'content_type': 'type'}]
        self.mock_list_s3.side_effect = [
            (200, elements), (200, [])]
        req = swob.Request.blank(
            '/v1/AUTH_a/s3?format=json',
            environ={'__test__.status': '200 OK',
                     '__test__.body': '[]',
                     'swift.trans_id': 'id'})
        status, headers, body_iter = req.call_application(self.app)
        self.assertEqual(self.mock_shunt_swift.mock_calls, [])
        self.mock_list_s3.assert_has_calls([
            mock.call('', 10000, '', ''),
            mock.call(u'unicod\xc3\xa9'.encode('utf-8'), 10000, '', '')])
        results = json.loads(body_iter)
        for i, entry in enumerate(results):
            self.assertEqual(elements[i], entry)

    def test_list_container_accept_json(self):
        elements = [{'name': 'abc',
                     'hash': 'ffff',
                     'bytes': 42,
                     'last_modified': 'date',
                     'content_type': 'type'},
                    {'name': u'unicod\xc3\xa9',
                     'hash': 'ffff',
                     'bytes': 1000,
                     'last_modified': 'date',
                     'content_type': 'type'}]
        self.mock_list_s3.side_effect = [
            (200, elements), (200, [])]
        req = swob.Request.blank(
            '/v1/AUTH_a/s3',
            environ={'__test__.status': '200 OK',
                     '__test__.body': '[]',
                     'swift.trans_id': 'id'},
            headers={'Accept': 'application/json'})
        status, headers, body_iter = req.call_application(self.app)
        self.assertEqual(self.mock_shunt_swift.mock_calls, [])
        self.mock_list_s3.assert_has_calls([
            mock.call('', 10000, '', ''),
            mock.call(u'unicod\xc3\xa9'.encode('utf-8'), 10000, '', '')])
        results = json.loads(body_iter)
        for i, entry in enumerate(results):
            self.assertEqual(elements[i], entry)

    @mock.patch('s3_sync.shunt.create_provider')
    def test_list_container_shunt_all_containers(self, create_mock):
        create_mock.return_value = mock.Mock()
        create_mock.return_value.list_objects.return_value = (200, [])
        req = swob.Request.blank(
            '/v1/AUTH_b/s3',
            environ={'__test__.status': '200 OK',
                     '__test__.body': '[]',
                     'swift.trans_id': 'id'})
        status, headers, body_iter = req.call_application(self.app)
        create_mock.assert_called_once_with({
            'account': 'AUTH_b',
            'container': 's3',
            'propagate_delete': False,
            'aws_bucket': 'dest-bucket',
            'aws_identity': 'user',
            'aws_secret': 'key'}, max_conns=1, per_account=True)

        # Follow it up with another request to a *different* container to make
        # sure we didn't bleed state
        create_mock.reset_mock()
        req = swob.Request.blank(
            '/v1/AUTH_b/s4',
            environ={'__test__.status': '200 OK',
                     '__test__.body': '[]',
                     'swift.trans_id': 'id'})
        status, headers, body_iter = req.call_application(self.app)
        create_mock.assert_called_once_with({
            'account': 'AUTH_b',
            'container': 's4',
            'propagate_delete': False,
            'aws_bucket': 'dest-bucket',
            'aws_identity': 'user',
            'aws_secret': 'key'}, max_conns=1, per_account=True)

    def test_list_container_shunt_swift(self):
        self.mock_list_swift.side_effect = [
            (200, [{'name': 'abc',
                    'hash': 'ffff',
                    'bytes': 42,
                    'last_modified': 'date',
                    'content_type': 'type'},
                   {'name': u'unicod\xe9',
                    'hash': 'ffff',
                    'bytes': 1000,
                    'last_modified': 'date',
                    'content_type': 'type'}]),
            (200, [])]
        req = swob.Request.blank(
            '/v1/AUTH_a/sw\xc3\xa9ft',
            environ={'__test__.status': '200 OK',
                     '__test__.body': '[]',
                     'swift.trans_id': 'id'})
        status, headers, body_iter = req.call_application(self.app)
        self.assertEqual(self.mock_shunt_swift.mock_calls, [])
        self.mock_list_swift.assert_has_calls([
            mock.call('', 10000, '', ''),
            mock.call(u'unicod\xe9'.encode('utf-8'), 10000, '', '')])
        names = body_iter.split('\n')
        self.assertEqual(['abc', u'unicod\xe9'], names)


from drf_yasg.utils import swagger_auto_schema
from rest_framework.generics import RetrieveUpdateAPIView, get_object_or_404
from rest_framework.response import Response
from rest_framework.views import APIView

from authentication.models import AppUser
from core.commonResponses import invalidDataResponse, successResponse
from core.commonSchemas import not_found_schema, invalid_data_schema, success_schema
from userprofile.schemas import follow_schema, business_schema
from userprofile.serializers import FullUserSerializer, AddImageSerializer
from userprofile.utils import add_profile_image, follow_user, toggle_business

class ProfileView(RetrieveUpdateAPIView):
    queryset = AppUser.objects.all()
    serializer_class = FullUserSerializer

class AddProfileImageView(APIView):
    @swagger_auto_schema(
        request_body=AddImageSerializer,
        responses={
            200: success_schema(),
            404: not_found_schema(),
            406: invalid_data_schema()
        }
    )
    def post(self, request):
        serializer = AddImageSerializer(data=request.data)

        if not serializer.is_valid():
            return invalidDataResponse()

        add_profile_image(request.user, serializer.validated_data['profile_image_id'])

        return successResponse()

class FollowUserView(APIView):
    @swagger_auto_schema(
        responses={
            200: follow_schema(),
            404: not_found_schema(),
        },
    )
    def put(self, request, pk):
        user = get_object_or_404(AppUser.objects.all(), id=pk)

        follow_response = follow_user(user, request.user)

        return Response({"follow": follow_response}, status=200)

class ToggleBusinessView(APIView):
    @swagger_auto_schema(
        responses={
            200: business_schema(),
            404: not_found_schema(),
        },
    )
    def put(self, request):
        business_response = toggle_business(request.user)

        return Response({"business": business_response}, status=200)


import torch
import torch.nn as nn
import torch
from torch.autograd import Variable
import copy
import torch.nn.functional as F
from torch.nn import BCELoss, MSELoss, CrossEntropyLoss

class RobertaClassificationHead(nn.Module):
    """Head for sentence-level classification tasks."""

    def __init__(self, config, args):
        super().__init__()
        self.dense = nn.Linear(config.hidden_size, config.hidden_size)
        self.dropout = nn.Dropout(config.hidden_dropout_prob)
        self.out_proj = nn.Linear(config.hidden_size, 1)
        self.args = args

    def forward(self, features, **kwargs):
        x = features[:, 0, :]  # take <s> token (equiv. to [CLS])
        x = self.dropout(x)
        x = self.out_proj(x)
        return x

class Model(nn.Module):
    def __init__(self, encoder, config, tokenizer, args):
        super(Model, self).__init__()
        self.encoder = encoder
        self.config = config
        self.tokenizer = tokenizer
        self.classifier = RobertaClassificationHead(config, args)
        self.args = args

    def forward(self, inputs_ids, attn_masks,
                labels=None):
        outputs = \
            self.encoder(input_ids=inputs_ids, attention_mask=attn_masks)[0]
        logits = self.classifier(outputs)

        prob = torch.sigmoid(logits)
        if labels is not None:
            loss_fct = BCELoss()
            loss = loss_fct(prob, labels.unsqueeze(1).float())
            return loss, prob
        else:
            return prob


# This is a sample Python script.

# Press ⌃R to execute it or replace it with your code.
# Press Double ⇧ to search everywhere for classes, files, tool windows, actions, and settings.

import fasttext
from tqdm import tqdm
# from lib.cut_sentence
from openpyxl import load_workbook
import sacremoses
import sentencepiece as spm

fasttext.FastText.eprint = lambda x: None
# 标签位置  1/前   2/后
label_postion = 2
# bpe模型
bpe_model = spm.SentencePieceProcessor()
bpe_model.Load("./bpe_en.model")
# 标签映射
label_dic = {
    '质量-其他': 'Quality_Other',
    '质量-破损': 'Quality_Broken',
    '投递-异常': 'Delivery_Exception',
    '质量-少配件': 'Quality_Few_Ingredients',
    '投递-咨询': 'Delivery_Consultation',
    '投递-变更': 'Delivery_Change',
    '订单-订单取消': 'Order_Order_Cancellation',
    '发票-发票': 'Invoice',
    '订单-订单变更': 'Order_Order_Changes',
    '投递-异常、退款': 'Delivery_Abnormal',
    '发票-发票变更': 'Invoice_Invoice_Change',
    '质量-产品破损': 'Quality_Product_Breakage',
    '其他': 'Other'
}

def token_normalizer(data_path, out_path):
    nor_en = sacremoses.MosesPunctNormalizer(lang='en')
    tok_en = sacremoses.MosesTokenizer(lang='en')

    writer = open(out_path, "a")

    for line in open(data_path).readlines():
        en_nor = nor_en.normalize(line)
        en_tok = ' '.join(tok_en.tokenize(en_nor))
        writer.write(en_tok)

def create_ru_subword_dict(data_path):
    spm.SentencePieceTrainer.Train(input=data_path, model_prefix='sub_word_ru', vocab_size=32000)

def sub_word(line):
    encoder = bpe_model
    tokens = []
    for token, ids in list(zip(encoder.Encode(line.strip(), out_type=str), encoder.Encode(line.strip(), out_type=int))):
        if encoder.IsUnknown(ids):
            tokens += list(token)
        else:
            tokens.append(token)
    return ' '.join(tokens)

def print_hi(name):
    # Use a breakpoint in the code line below to debug your script.
    print(f'Hi, {name}')  # Press ⌘F8 to toggle the breakpoint.

def process_excel_data(file, input_data, bpe=False):
    excel = load_workbook(input_data)
    table = excel[excel.sheetnames[0]]
    for i in table.rows:
        label = label_dic.get(i[1].value)
        data = i[0].value.replace("\n", " ")
        if bpe:
            data = sub_word(data)
        if label_postion == 1:
            file.write("__label__{} \t {} \n".format(label, data))
        else:
            file.write("{} \t__label__{}\n".format(data, label))

def model_train_old(model_path, data_path):
    model = fasttext.load_model(model_path)
    model = model.train_supervised(input=data_path, epoch=25, lr=0.6, wordNgrams=2, verbose=2, minCount=1)
    model.save_model(model_path)

def train(model_path, data_path, auto_arg=False, validation_file='', auto_time=600):
    if auto_arg:
        new_model = fasttext.train_supervised(input=data_path, autotuneValidationFile=validation_file,
                                              autotuneDuration=auto_time)
    else:
        new_model = fasttext.train_supervised(input=data_path, epoch=25, lr=0.6, wordNgrams=4, verbose=2, minCount=1)
    new_model.save_model(model_path)

def model_test(model_path, data_path):
    model = fasttext.load_model(model_path)
    res = model.test(data_path)
    print(res)

def model_red(model_path, data_path):
    model = fasttext.load_model(model_path)
    print(model.words)
    list = []
    for line in open(data_path).readlines():
        list.append(line.replace("\n", ""))
    res = model.predict(list)

    for i in res:
        print(i)

    # Press the green button in the gutter to run the script.

if __name__ == '__main__':
    # 训练集  普通训练集，BPE训练集
    # 测试集  普通测试集，BPE测试集

    # 定参训练      普通训练 BPE参数
    # 自动超参训练   普通训练 BPE训练 20分钟

    # 测试 定参 普通、BPE
    # 测试 自动 普通、BPE

    # 数据整理 测试集
    # process_excel_data(open('./test', "a"), "./test.xlsx")

    # 数据整理 PBE数据集
    # process_excel_data(open('./data_bpe', "w"), "./train.xlsx", True)
    # 数据整理 PBE测试集
    # process_excel_data(open('./test_bpe', "w"), "./test.xlsx", True)

    # 模型训练
    # train(r"./m1.bin", "./data")
    # 模型训练 bpe  手动参数
    # train(r"./m1_bpe_m.bin", "./data_bpe")
    # 数据训练 bpe  自动搜索
    # train(r"./m1_pbe.bin", "./data_bpe", True, "./test_bpe")
    # 模型测试
    # model_test("./m1.bin", "./test")
    # 模型测试 bpe
    # model_test("./m1_bpe.bin", "./test_bpe")
    # 模型测试 bpe 手工参数
    # model_test("./m1_bpe_m.bin", "./test_bpe")
    # 模型预测
    # model_red("./m1.bin", "./test")

    model = fasttext.load_model("./m1.bin")
    print(model.words)


# Pickleによるモデル保存
import pandas as pd
from sklearn import model_selection
from sklearn.neural_network import MLPClassifier
import pickle

def save(X_train,Y_train):
    # モデルをトレーニングする
    model = MLPClassifier(solver="sgd",random_state=0,max_iter=10000)
    model.fit(X_train, Y_train)

    # モデルを保存する
    filename = '/Users/shoda/labo/change/php/finalized_model.sav'
    pickle.dump(model, open(filename, 'wb'))

if __name__ == '__main__':
    df = pd.read_csv("/Users/shoda/labo/learning/data/loading/binary.csv",index_col=0) # バイナリ
    X = df.drop('result', axis=1)
    Y = df["result"]
    save(X,Y)


import pygame

pygame.init() # 초기화 반드시 필요 -> 'pygame' 사용 시 반드시 선언

#화면 크기 설정
screen_width = 480 #가로
screen_height = 640 #세로
screen = pygame.display.set_mode((screen_width, screen_height))

#화면 타이틀 설정
pygame.display.set_caption("Game Project") #게임 이름

#배경 이미지 불러오기
background = pygame.image.load("C:\\devtools\\pythonworkspace\\first_python_project\\background.png")

#이벤트 루프
running = True #게임 진행 여부
while running:
    #이 부분도 'pygame' 사용 시 반드시 필요하다 몰라도 된다. 적기만 하자
    for event in pygame.event.get(): #event.get()을 통해 사용자의 움직임을 받음
        if event.type == pygame.QUIT: #창의 x버튼
            running = False

    screen.blit(background, (0, 0)) #x, y 좌표로 blit()가 배경을 그린다.
    pygame.display.update() #화면을 계속해서 그려주는 함수

# 게임 종료시 pygame 종료
pygame.quit()


def evenOneven(getal):
    l = list(str(getal))
    tellerEven = tellerOneven = 0

    for i in range (0, len(l)):
        if int(l[i]) % 2 == 1:
            tellerOneven += 1
        #elif int(l[i]) == 0:
        #    pass
        elif int(l[i]) % 2 == 0:
            tellerEven += 1
    return (tellerEven, tellerOneven)

def volgende(getal):
    l = list(str(getal))
    aantalGetallen = len(l)
    tellerEven = tellerOneven = 0

    for i in range (0, aantalGetallen):
        if int(l[i]) % 2 == 1:
            tellerOneven += 1
        elif int(l[i]) % 2 == 0:
            tellerEven += 1

    return int(str(tellerEven) + str(tellerOneven) + str(aantalGetallen))

def stappen(getal):
    teller = 0
    while getal != 123:
        getal = volgende(getal)
        teller += 1
    return teller


# -*- coding: utf-8
import pandas as pd
import dask.dataframe as dd
from pprint import pprint

pd.set_option('display.expand_frame_repr', False)

def transform_one(series):
    """Takes a Series object representing a grouped dataframe row,
    and returns a string (serialized JSON).

    :return: dict
    """
    data = series.to_dict()
    if not data:
        return pd.Series([], name='data')
    (customer, url, ts, _) = data.pop('index')
    page_views = data.pop('views')
    visitors = data.pop('visitors')
    data.update({
        'customer': customer,
        'url': url,
        'ts': ts.strftime('%Y-%m-%dT%H:%M:%S'),
        'metrics': {'views': page_views, 'visitors': visitors}
    })
    return pd.Series([data], name='data')

if __name__ == '__main__':
    pdf = pd.DataFrame.from_records([
        ("http://a.com/articles/1", "http://google.com/", "xxx", "2017-09-15 00:10:00", "a.com"),
        ("http://a.com/articles/2", "http://bing.com/", "yyy", "2017-09-15 00:20:00", "a.com"),
        ("http://a.com/articles/2", "http://facebook.com/", "yyy", "2017-09-15 00:30:00", "a.com"),
        ("http://a.com/articles/1", "http://google.com/", "xxx", "2017-09-15 01:10:00", "a.com"),
        ("http://a.com/articles/2", "http://bing.com/", "yyy", "2017-09-15 01:20:00", "a.com")
    ],
    columns=['url', 'referrer', 'session_id', 'ts', 'customer'])

    pdf['ts'] = pd.to_datetime(pdf['ts'])

    df = dd.from_pandas(pdf, 2)

    # round datetimes down to an hour
    df['ts'] = df['ts'].dt.floor('1H')

    # group on customer, timestamp (rounded) and url
    gb = df.groupby(['customer', 'url', 'ts'])

    df = gb.apply(lambda d: pd.DataFrame({
            'views': len(d),
            'visitors': d.session_id.count(),
            'referrers': [d.referrer.tolist()]}),
        meta={'views': int, 'visitors': int, 'referrers': int})

    # I want index to be a part of dataframe, so it is passed
    # to the next .apply call.
    df['index'] = df.index

    df = df.apply(transform_one, axis=1, meta={'data': str}).to_bag()

    pprint(df.compute())


import cv2
import numpy as np

img = cv2.imread("unidade 01\decomposicao_img_bits\imgs\desafio-esteganografia.png")

if img is None:
    print("Erro ao abrir a imagem")
    exit()

img_carrier = np.copy(img)
img_encoded = np.copy(img)
nbits = 3

img_carrier = img >> nbits << nbits
img_encoded = img << (8 - nbits)

cv2.imshow("Imagem portadora", img_carrier)
cv2.imshow("Imagem codificada", img_encoded)
cv2.waitKey()


#!/usr/bin/python
# -*- coding: utf-8 -*-

import json
with open ('author-extract.json') as f:
	data = json.loads(f.read())

f = open('ready4cartodb.csv','w+')
f.write('id, author, latitude, longitude\n')
for record in data['records']:
	for author in record['authors']:
		if 'latitude' in author.keys():
			f.write(str(author['id']) + ', ' + author['author'].replace(',',' ') + ', ' + str(author['latitude'])+ ', ' + str(author['longitude'])+'\n')


from typing import Any, Dict, List, Type, TypeVar, Union

import attr

from ..types import UNSET, Unset

T = TypeVar("T", bound="ScimServiceProviderConfigBulk")

@attr.s(auto_attribs=True)
class ScimServiceProviderConfigBulk:
    """
    Attributes:
        supported (Union[Unset, bool]):
        max_operations (Union[Unset, int]):
        max_payload_size (Union[Unset, int]):
    """

    supported: Union[Unset, bool] = UNSET
    max_operations: Union[Unset, int] = UNSET
    max_payload_size: Union[Unset, int] = UNSET
    additional_properties: Dict[str, Any] = attr.ib(init=False, factory=dict)

    def to_dict(self) -> Dict[str, Any]:
        supported = self.supported
        max_operations = self.max_operations
        max_payload_size = self.max_payload_size

        field_dict: Dict[str, Any] = {}
        field_dict.update(self.additional_properties)
        field_dict.update({})
        if supported is not UNSET:
            field_dict["supported"] = supported
        if max_operations is not UNSET:
            field_dict["maxOperations"] = max_operations
        if max_payload_size is not UNSET:
            field_dict["maxPayloadSize"] = max_payload_size

        return field_dict

    @classmethod
    def from_dict(cls: Type[T], src_dict: Dict[str, Any]) -> T:
        d = src_dict.copy()
        supported = d.pop("supported", UNSET)

        max_operations = d.pop("maxOperations", UNSET)

        max_payload_size = d.pop("maxPayloadSize", UNSET)

        scim_service_provider_config_bulk = cls(
            supported=supported,
            max_operations=max_operations,
            max_payload_size=max_payload_size,
        )

        scim_service_provider_config_bulk.additional_properties = d
        return scim_service_provider_config_bulk

    @property
    def additional_keys(self) -> List[str]:
        return list(self.additional_properties.keys())

    def __getitem__(self, key: str) -> Any:
        return self.additional_properties[key]

    def __setitem__(self, key: str, value: Any) -> None:
        self.additional_properties[key] = value

    def __delitem__(self, key: str) -> None:
        del self.additional_properties[key]

    def __contains__(self, key: str) -> bool:
        return key in self.additional_properties


from restful_backend.settings.base import * #pylint: disable=unused-wildcard-import
import dj_database_url
import django_heroku

'''
Heroku specific settings for production
'''

# SECURITY WARNING: keep the secret key used in production secret!
SECRET_KEY = os.environ.get('SECRET_KEY')
DATABASE_URL = os.environ.get('DATABASE_URL')

# SECURITY WARNING: don't run with debug turned on in production!
DEBUG = False

# Django security checklist
SECURE_CONTENT_TYPE_NOSNIFF = True
SECURE_BROWSER_XSS_FILTER = True
SESSION_COOKIE_SECURE = True
CSRF_COOKIE_SECURE = True
SECURE_SSL_REDIRECT = True
X_FRAME_OPTIONS = 'DENY'

ALLOWED_HOSTS = ['localhost']

REST_FRAMEWORK = {

    'DEFAULT_AUTHENTICATION_CLASSES': (
        'rest_framework_simplejwt.authentication.JWTAuthentication',
    ),
    'DEFAULT_RENDERER_CLASSES': [
        'rest_framework.renderers.JSONRenderer',
    ]
}

# Database
# https://docs.djangoproject.com/en/2.2/ref/settings/#databases

DATABASES = {
    'default': dj_database_url.config(default=DATABASE_URL)
}

django_heroku.settings(locals())


from queue import Queue
from .lep_time import get_now_origin
import copy

class LEPQUEUE_ERROR(Exception): pass

class Lep_QueueMap:
    # 静态变量
    map = {}
    def __init__(self, ownername:str, owner) -> None:
        setattr(owner, 'qmap', self)
        self.owner = owner
        self.ownername:str = ownername

    def exist_name_check(self, name):
        if Lep_QueueMap.map.__contains__(name): raise LEPQUEUE_ERROR(f'队列中已存在此命名{name}')

    def not_exist_name_check(self, name):
        if not Lep_QueueMap.map.__contains__(name): raise LEPQUEUE_ERROR(f'队列中不存在此命名{name}')

    def add(self, name:str, maxsize:int = 0):
        self.exist_name_check(name)
        Lep_QueueMap.map[name] =  Queue(maxsize)

    def putfn(self, name:str, fnname: str, usercopy:bool = False, *para_list, **para_dict):
        self.not_exist_name_check(name)
        if usercopy:
            copy_para_list = copy.deepcopy(para_list)
            copy_para_dict = copy.deepcopy(para_dict)
        else:
            copy_para_list = para_list
            copy_para_dict = para_dict
        Lep_QueueMap.map[name].put({'fnname':fnname, 'para_list': copy_para_list, 'para_dict': copy_para_dict, 'time': get_now_origin()})

    def getfnall(self):
        self.not_exist_name_check(self.ownername)
        q:Queue = Lep_QueueMap.map[self.ownername]
        while not q.empty():
            act:dict = q.get()
            fn = getattr(self.owner, act['fnname'], None)
            fn(*act['para_list'], **act['para_dict'])


from formalchemy import FieldSet
from flaskext.wtf import Form, TextField, Required, TextAreaField, FloatField

from cfmi.billing.models import Session, Problem

class SessionForm(FieldSet):
    def __init__(self):
        FieldSet.__init__(self, Session)
        self.configure(options=[
                self.sched_start.label("Scheduled Start"),
                self.sched_end.label("Scheduled End"),
                self.approved.label("Approved?"),
                self.cancelled.label("Cancelled?"),
                self.start.label("Actual Start"),
                self.end.label("Actual End"),
                self.notes.label("Log").textarea()], exclude=[
                self.problem])

class ROSessionForm(SessionForm):
    def __init__(self):
        SessionForm.__init__(self)
        self.configure(readonly=True)

class ProblemForm(FieldSet):
    def __init__(self):
        FieldSet.__init__(self, Problem)
        self.configure(exclude=[self['session']])

class ProblemRequestForm(Form):
    problem = TextAreaField("Describe the nature of our error:", [Required()])
    duration = FloatField("What is the accurate duration, in hours, of this scan?", [Required()])


import logging

from fastapi import APIRouter, Depends
from ..db import DB

from openaq_fastapi.models.responses import (
    OpenAQResult,
)

logger = logging.getLogger("manufacturers")
logger.setLevel(logging.DEBUG)

router = APIRouter()

@router.get("/v2/manufacturers", response_model=OpenAQResult, tags=["v2"])
async def mfr_get(db: DB = Depends()):

    q = """
    WITH t AS (
    select distinct metadata->>'manufacturer_name' as vals from sensor_systems
    where metadata ? 'manufacturer_name'
    )
    SELECT count(*) OVER () as count, to_jsonb(vals) as json FROM t
    """

    output = await db.fetchOpenAQResult(q, {"page": 1, "limit": 1000})

    return output

@router.get("/v2/models", response_model=OpenAQResult, tags=["v2"])
async def model_get(db: DB = Depends()):

    q = """
    WITH t AS (
    select distinct metadata->>'model_name' as vals from sensor_systems
    where metadata ?'model_name'
    )
    SELECT count(*) OVER () as count, to_jsonb(vals) as json FROM t
    """

    output = await db.fetchOpenAQResult(q, {"page": 1, "limit": 1000})

    return output


from rest_framework import status
from rest_framework.response import Response
from rest_framework.viewsets import ViewSet

from api.models import Stock, UserStock
from api.permissions import IsInvestor
from api.serializers import StockTransactionSerializer

class BuyStockTransactionViewSet(ViewSet):
    serializer_class = StockTransactionSerializer
    permission_classes = [IsInvestor]

    def create(self, request):
        stock_id = request.data.get('stockId')
        quantity = int(request.data.get('quantity'))

        try:
            stock = Stock.objects.get(pk=stock_id)
        except Stock.DoesNotExist:
            return Response({'error': 'Stock not found'}, status=status.HTTP_404_NOT_FOUND)

        if stock.quantity < quantity:
            return Response({'error': 'There is not enough stock to purchase this quantity.'},
                            status=status.HTTP_400_BAD_REQUEST)

        user_stock, created = UserStock.objects.get_or_create(user=request.user, stock=stock)
        user_stock.quantity += quantity
        user_stock.save()

        stock.quantity -= quantity
        stock.save()

        data = {
            'stockId': stock_id,
            'quantity': quantity,
            'totalCost': stock.price * quantity
        }

        return Response(data, status=status.HTTP_201_CREATED)

class SellStockTransactionViewSet(ViewSet):
    serializer_class = StockTransactionSerializer
    permission_classes = [IsInvestor]

    def create(self, request):
        stock_id = request.data.get('stockId')
        quantity = int(request.data.get('quantity'))

        try:
            stock = Stock.objects.get(pk=stock_id)
        except Stock.DoesNotExist:
            return Response({'error': 'Stock not found'}, status=status.HTTP_404_NOT_FOUND)

        try:
            user_stock = UserStock.objects.get(user=request.user, stock=stock)
        except UserStock.DoesNotExist:
            return Response({'error': 'User does not hold this stock'}, status=status.HTTP_404_NOT_FOUND)

        if quantity > user_stock.quantity:
            return Response({'error': 'Cannot sell more stock than the user holds'}, status=status.HTTP_400_BAD_REQUEST)

        user_stock.quantity -= quantity
        user_stock.save()

        stock.quantity += quantity
        stock.save()

        data = {
            'stockId': stock_id,
            'quantity': quantity,
            'totalProfit': stock.price * quantity
        }

        return Response(data, status=status.HTTP_201_CREATED)


from skimage.metrics import structural_similarity
import cv2
import numpy as np
import argparse
import os
SSIM_RES_FILE = 'dump/ssim.txt'

def compute_rise_ssim(uncal_img, cal_img):
    """
    """
    # cal = cv2.imread(args.cal)
    # uncal = cv2.imread(args.unc)
    (score, diff) = structural_similarity(cv2.imread(uncal_img), cv2.imread(cal_img), full=True, multichannel=True)
    with open(SSIM_RES_FILE, 'a') as f:
        f.writelines(str(round(score, 3)) + '\n')
    f.close()
    return score

if __name__ == "__main__":
    parser = argparse.ArgumentParser(description='Training params')
    parser.add_argument('--unc', type=str, default='/u/dw3zn/Repos/saliency_calibration/rise_uncalibrated.png',
                        help='Uncalibrated saliency map')
    parser.add_argument('--cal', type=str, default='/u/dw3zn/Repos/saliency_calibration/rise_calibrated.png',
    help='Calibrated saliency map')
    args = parser.parse_args()
    compute_rise_ssim()


from flask import Flask, render_template, request
import matplotlib.pyplot as plt
import matplotlib as mpl
from matplotlib.font_manager import FontProperties

# 绘制饼图时使用中文字体

app = Flask(__name__)

# 指定中文字体
font = FontProperties(fname=r"/libraries/font-apex/2.0/fonts/Font-APEX-Large.ttf")

@app.route('/', methods=['GET', 'POST'])
def calculate():
    if request.method == 'POST':
        input_value = float(request.form['input_name'])  # 获取输入值
        # 计算比例
        life_expenses = input_value * 0.55
        savings_investment = input_value * 0.35
        risk_management = input_value * 0.1

        # 绘制饼图
        mpl.rcParams['font.family'] = ['Arial Unicode MS']
        labels = ['生活開支', '儲蓄與投資', '風險管理']
        sizes = [life_expenses, savings_investment, risk_management]
        colors = ['red', 'yellow', 'orange']
        plt.pie(sizes, labels=labels, colors=colors, autopct='%1.1f%%')
        # plt.title('pie')

        # 将图表保存为图片文件
        chart_path = './static/chart.png'  # 图表保存路径，相对于当前工作目录
        plt.savefig(chart_path, format='png')
        plt.close()

        return render_template('index.html', result=[life_expenses, savings_investment, risk_management], chart_path=chart_path)

    return render_template('index.html', result=None, chart_path=None)

if __name__ == '__main__':
    app.run()


class Solution:
    def findRotation(self, mat: List[List[int]], target: List[List[int]]) -> bool:
        def rotateBy90(matrix):
            all_same = True
            new_mat = deepcopy(matrix)
            for i in range(len(new_mat)):
                row = matrix[i]
                for j in range(len(matrix[0])):
                    new_mat[j][len(new_mat) - i- 1] = row[j]
                    if target[j][len(new_mat) -i- 1] != row[j]:
                        all_same = False
            return new_mat, all_same
        fir, g = rotateBy90(mat)
        if g:
            return True
        sec,g = rotateBy90(fir)
        if g:
            return True
        thr,g = rotateBy90(sec)
        if g:
            return True
        four,g = rotateBy90(thr)
        if g:
            return True
        return False


import sys
import socket
import logging
import ssl
import os
from bs4 import BeautifulSoup

import gzip

BUFFER_SIZE = 1024 * 10
SERVER_HOST = "localhost"

def create_socket(destination_address=SERVER_HOST, port=12000):
    try:
        sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
        server_address = (destination_address, port)
        logging.warning(f"connecting to {server_address}")
        sock.connect(server_address)
        return sock
    except Exception as ee:
        logging.warning(f"error {str(ee)}")

def create_secure_socket(destination_address=SERVER_HOST, port=10000):
    try:
        # get it from https://curl.se/docs/caextract.html
        sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
        server_address = (destination_address, port)
        logging.warning(f"connecting to {server_address}")
        sock.connect(server_address)
        secure_socket = ssl.wrap_socket(sock, keyfile=None, certfile=None, server_side=False,
                                        cert_reqs=ssl.CERT_NONE, ssl_version=ssl.PROTOCOL_SSLv23)
        logging.warning(secure_socket.getpeercert())
        return secure_socket
    except Exception as ee:
        logging.warning(f"error {str(ee)}")

def send_command(server, command_str, is_secure=False):
    request_path = command_str.split("\r\n")[0].split("GET")[
        1].split("HTTP")[0].strip()
    print("request_path", request_path)

    headers = ""
    content_encoded = b""
    content = ""
    content_encoding = "utf-8"
    content_length = -1
    content_type = "text/html"

    bytes_received = 0

    server_address = server[0]
    server_port = server[1]
    sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
    if is_secure == True:
        sock = create_secure_socket(server_address, server_port)
    else:
        sock = create_socket(server_address, server_port)

    try:
        logging.warning(f"[+] Sending data ")
        sock.sendall(command_str.encode())
        while True:
            # socket does not receive all data at once, data comes in part, need to be concatenated at the end of process
            data = sock.recv(BUFFER_SIZE)
            if data:
                if "html" not in content_type:
                    bytes_received += len(data)
                    print(
                        f"[!] Bytes Received {bytes_received}/{content_length}")

                    save_file_name = request_path.split("/")[-1]
                    file = open(save_file_name, "ab")
                    file.write(data)
                    file.close()

                    if bytes_received == content_length:
                        content = f""
                        break
                    continue

                decoded_data = data.decode().split("\r\n\r\n")
                headers += decoded_data[0]
                content += decoded_data[1]

                # read content-encoding and content-length
                for head in headers.split("\r\n"):
                    if "Content-Encoding:" in head:
                        content_encoding = head.split(
                            "Content-Encoding: ")[1]
                    if "Content-Length:" in head:
                        content_length = int(
                            head.split("Content-Length: ")[1])
                    if "Content-Type:" in head:
                        content_type = head.split(
                            "Content-Type: ")[1].strip()

                logging.warning(f"Content-Encoding: {content_encoding}")
                logging.warning(f"Content-Length: {content_length}")
                logging.warning(f"Content-Type: {content_type}")

                if "html" not in content_type:
                    try:
                        save_file_name = request_path.split("/")[-1]
                        os.remove(save_file_name)
                    except:
                        None

                if len(content) == content_length:
                    if content_encoding == "gzip":
                        content = gzip.decompress(content_encoded).decode()
                    elif content_encoding == "utf-8":
                        None
                    else:
                        content = "[Downloadable Content!]"
                    break
            else:
                print("[!] No More Data Received")
                break

        logging.warning("[+] All data has been received")
        return headers, content
    except Exception as ee:
        logging.warning(f"[!] Error during data receiving {str(ee)}")
        return False

def create_request_headers(req_str):
    data = ""
    for head in req_str.split("\n"):
        if head != "":
            data += f"{head}\r\n"

    data += "\r\n"
    return data

#
# START
#

try:
    while True:
        print(f"[!] Where to?: http://{SERVER_HOST}/", end="")
        dest = input()
        request_headers = f"""
GET /{dest} HTTP/1.1
Host: {SERVER_HOST}
User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/99.0.4844.82 Safari/537.36
"""
        headers, content = send_command(
            (SERVER_HOST, 8000), create_request_headers(request_headers), False)
        print("[!] HEADERS:")
        print(headers)
        soup = BeautifulSoup(content, "html.parser")
        html_text = soup.get_text()
        print(html_text)

except KeyboardInterrupt:
    sys.exit(0)


from datetime import datetime
from logging import getLogger

from activities.models import Activity
from activities.models import ActivityReply
from django.core.management.base import BaseCommand
from django.db.models.aggregates import Count

logging = getLogger(__name__)

class Command(BaseCommand):
    help = "Updates activity reply counts for all activities."

    def add_arguments(self, parser):
        parser.add_argument('--activity_ids',
                            nargs='+',
                            dest='activity_ids',
                            default=None,
                            type=int,
                            help=('An acitiity id or an space separated list '
                                  'of activity ids to update the reply counts '
                                  'for. If None, this will update all activity '
                                  'reply counts.'))
        parser.add_argument('-d', '--dry_run',
                            dest='dry_run',
                            default=False,
                            type=bool,
                            help=('boolean indicating if the images should '
                                  'actually be processed or if the function '
                                  'out just wants to be seen.'))

    def handle(self, activity_ids=None, dry_run=False, *args, **options):
        """
        :param activity_ids: list of activity id to update the reply counts for.
            If None, this will process all activities.
        :param dry_run: boolean indicating if the objects should actually be
            processed or if the function out just wants to be seen.
        """
        if dry_run == True:
            logging.info('"dry_run" has been set to true. No actual '
                         'updates will be made.')

        start = datetime.utcnow()

        # get all activity ids that have replies
        activity_ids = (ActivityReply.objects.all()
                                             .order_by('activity')
                                             .distinct('activity')
                                             .values_list('activity',
                                                          flat=True))

        logging.info('Updating all activities with no replies to have a '
                     'reply_count of 0...')

        if not dry_run:
            # Update all reply counts for other activities to 0
            Activity.objects.exclude(id__in=activity_ids).update(reply_count=0)

        queryset = Activity.objects.filter(
            id__in=activity_ids
        ).annotate(reply_count_actual=Count('replies'))

        logging.info('Starting to update activities...')

        # update all activities will replies to the correct count
        for activity in queryset:
            activity.reply_count = activity.reply_count_actual
            msg = 'Updating activity id: "{0}" with reply count of "{1}"'.format(
                activity.id,
                activity.reply_count_actual
            )
            logging.info(msg)

            if not dry_run:
                activity.save()

        end = datetime.utcnow()
        total_seconds = (end - start).seconds
        logging.info('Updated {0} activities reply counts in {1} seconds!'.format(
            len(activity_ids),
            total_seconds
        ))


import jittor as jt
import jittor.nn as nn

from jdet.utils.registry import HEADS, LOSSES, build_from_cfg
from jdet.utils.general import multi_apply
from jdet.ops.bbox_transfomrs import bbox2delta, mask2poly, get_best_begin_point, polygonToRotRectangle_batch, hbb2obb_v2, dbbox2delta_v3, best_match_dbbox2delta, delta2dbbox_v3
from jdet.ops.nms_rotated import multiclass_nms_rotated
from jdet.models.boxes.box_ops import rotated_box_to_poly

def bbox_target_rbbox(pos_bboxes_list,
                neg_bboxes_list,
                pos_assigned_gt_inds_list,
                gt_masks_list,
                pos_gt_labels_list,
                cfg,
                reg_classes=1,
                target_means=[.0, .0, .0, .0, .0],
                target_stds=[1.0, 1.0, 1.0, 1.0, 1.0],
                concat=True,
                with_module=True,
                hbb_trans='hbb2obb_v2'):
    labels, label_weights, bbox_targets, bbox_weights = multi_apply(
        bbox_target_rbbox_single,
        pos_bboxes_list,
        neg_bboxes_list,
        pos_assigned_gt_inds_list,
        gt_masks_list,
        pos_gt_labels_list,
        cfg=cfg,
        reg_classes=reg_classes,
        target_means=target_means,
        target_stds=target_stds,
        with_module=with_module,
        hbb_trans=hbb_trans)

    if concat:
        labels = jt.contrib.concat(labels, 0)
        label_weights = jt.contrib.concat(label_weights, 0)
        bbox_targets = jt.contrib.concat(bbox_targets, 0)
        bbox_weights = jt.contrib.concat(bbox_weights, 0)
    return labels, label_weights, bbox_targets, bbox_weights

def bbox_target_rbbox_single(pos_bboxes,
                       neg_bboxes,
                       pos_assigned_gt_inds,
                       gt_masks,
                       pos_gt_labels,
                       cfg,
                       reg_classes=1,
                       target_means=[.0, .0, .0, .0, .0],
                       target_stds=[1.0, 1.0, 1.0, 1.0, 1.0],
                       with_module=True,
                       hbb_trans='hbb2obb_v2'):
    num_pos = pos_bboxes.size(0)
    num_neg = neg_bboxes.size(0)
    num_samples = num_pos + num_neg
    labels = jt.zeros(num_samples, dtype=jt.int)        #origin: torch.long
    label_weights = jt.zeros(num_samples)
    bbox_targets = jt.zeros((num_samples, 5))
    bbox_weights = jt.zeros((num_samples, 5))
    pos_gt_masks = gt_masks[pos_assigned_gt_inds]
    pos_gt_polys = mask2poly(pos_gt_masks)
    pos_gt_bp_polys = get_best_begin_point(pos_gt_polys)
    pos_gt_obbs = jt.array(polygonToRotRectangle_batch(pos_gt_bp_polys, with_module))

    if pos_bboxes.size(1) == 4:
        pos_ext_bboxes = hbb2obb_v2(pos_bboxes)
    else:
        pos_ext_bboxes = pos_bboxes

    if num_pos > 0:
        labels[:num_pos] = pos_gt_labels
        pos_weight = 1.0 if cfg['pos_weight'] <= 0 else cfg['pos_weight']
        label_weights[:num_pos] = pos_weight
        if with_module:
            pos_bbox_targets = dbbox2delta(pos_ext_bboxes, pos_gt_obbs, target_means,
                                          target_stds)
        else:
            pos_bbox_targets = dbbox2delta_v3(pos_ext_bboxes, pos_gt_obbs, target_means,
                                              target_stds)
        bbox_targets[:num_pos, :] = pos_bbox_targets
        bbox_weights[:num_pos, :] = 1
    if num_neg > 0:
        label_weights[-num_neg:] = 1.0

    return labels, label_weights, bbox_targets, bbox_weights

def rbbox_target_rbbox(pos_rbboxes_list,
                         neg_rbboxes_list,
                         pos_gt_rbboxes_list,
                         pos_gt_labels_list,
                         cfg,
                         reg_classes=1,
                         target_means=[.0, .0, .0, .0, 0],
                         target_stds=[1.0, 1.0, 1.0, 1.0, 1.0],
                         concat=True):
    labels, label_weights, bbox_targets, bbox_weights = multi_apply(
        rbbox_target_rbbox_single,
        pos_rbboxes_list,
        neg_rbboxes_list,
        pos_gt_rbboxes_list,
        pos_gt_labels_list,
        cfg=cfg,
        reg_classes=reg_classes,
        target_means=target_means,
        target_stds=target_stds)

    if concat:
        labels = jt.contrib.concat(labels, 0)
        label_weights = jt.contrib.concat(label_weights, 0)
        bbox_targets = jt.contrib.concat(bbox_targets, 0)
        bbox_weights = jt.contrib.concat(bbox_weights, 0)
    return labels, label_weights, bbox_targets, bbox_weights

def rbbox_target_rbbox_single(pos_rbboxes,
                       neg_rbboxes,
                       pos_gt_rbboxes,
                       pos_gt_labels,
                       cfg,
                       reg_classes=1,
                       target_means=[.0, .0, .0, .0, .0],
                       target_stds=[1.0, 1.0, 1.0, 1.0, 1.0]):
    assert pos_rbboxes.size(1) == 5
    num_pos = pos_rbboxes.size(0)
    num_neg = neg_rbboxes.size(0)
    num_samples = num_pos + num_neg
    labels = jt.zeros(num_samples, dtype=jt.int)        #origin: torch.long
    label_weights = jt.zeros(num_samples)
    bbox_targets = jt.zeros((num_samples, 5))
    bbox_weights = jt.zeros((num_samples, 5))

    if num_pos > 0:
        labels[:num_pos] = pos_gt_labels
        pos_weight = 1.0 if cfg.pos_weight <= 0 else cfg.pos_weight
        label_weights[:num_pos] = pos_weight
        pos_bbox_targets = best_match_dbbox2delta(pos_rbboxes, pos_gt_rbboxes, target_means, target_stds)
        bbox_targets[:num_pos, :] = pos_bbox_targets
        bbox_weights[:num_pos, :] = 1
    if num_neg > 0:
        label_weights[-num_neg:] = 1.0

    return labels, label_weights, bbox_targets, bbox_weights

def accuracy(pred, target, topk=1):
    if isinstance(topk, int):
        topk = (topk, )
        return_single = True
    else:
        return_single = False

    maxk = max(topk)
    _, pred_label = pred.topk(maxk, 1, True, True)
    pred_label = pred_label.t()
    correct = pred_label.equal(target.view(1, -1).expand_as(pred_label))

    res = []
    for k in topk:
        correct_k = correct[:k].view(-1).float().sum(0, keepdims=True)
#        res.append(correct_k.mul_(100.0 / pred.size(0)))
        correct_k *= 100.0 / pred.shape[0]
        res.append(correct_k)
    return res[0] if return_single else res

@HEADS.register_module()
class BBoxHeadRbbox(nn.Module):
    """Simplest RoI head, with only two fc layers for classification and
    regression respectively"""

    def __init__(self,
                 with_avg_pool=False,
                 with_cls=True,
                 with_reg=True,
                 roi_feat_size=7,
                 in_channels=256,
                 num_classes=19,
                 target_means=[0., 0., 0., 0., 0.],
                 target_stds=[0.1, 0.1, 0.2, 0.2, 0.1],
                 reg_class_agnostic=False,
                 with_module=True,
                 hbb_trans='hbb2obb_v2',
                 loss_cls=dict(
                     type='CrossEntropyLoss',
                     use_sigmoid=False,
                     loss_weight=1.0),
                 loss_bbox=dict(
                     type='SmoothL1Loss', beta=1.0, loss_weight=1.0)):
        super(BBoxHeadRbbox, self).__init__()
        assert with_cls or with_reg
        self.with_avg_pool = with_avg_pool
        self.with_cls = with_cls
        self.with_reg = with_reg
        self.roi_feat_size = roi_feat_size
        self.in_channels = in_channels
        self.num_classes = num_classes
        self.target_means = target_means
        self.target_stds = target_stds
        self.reg_class_agnostic = reg_class_agnostic

        self.loss_cls = build_from_cfg(loss_cls, LOSSES)
        self.loss_bbox = build_from_cfg(loss_bbox, LOSSES)

        in_channels = self.in_channels
        if self.with_avg_pool:
            self.avg_pool = nn.AvgPool2d(roi_feat_size)
        else:
            if isinstance(self.roi_feat_size, int):
                in_channels *= (self.roi_feat_size * self.roi_feat_size)
            elif isinstance(self.roi_feat_size, tuple):
                assert len(self.roi_feat_size) == 2
                assert isinstance(self.roi_feat_size[0], int)
                assert isinstance(self.roi_feat_size[1], int)
                in_channels *= (self.roi_feat_size[0] * self.roi_feat_size[1])
        if self.with_cls:
            self.fc_cls = nn.Linear(in_channels, num_classes)
        if self.with_reg:
            out_dim_reg = 5 if reg_class_agnostic else 5 * num_classes
            self.fc_reg = nn.Linear(in_channels, out_dim_reg)
        self.debug_imgs = None
        self.with_module = with_module
        self.hbb_trans = hbb_trans

    def init_weights(self):
        if self.with_cls:
#            nn.init.normal_(self.fc_cls.weight, 0, 0.01)
            nn.init.gauss_(self.fc_cls,0, 0.01)
            nn.init.constant_(self.fc_cls.bias, 0)
        if self.with_reg:
#            nn.init.normal_(self.fc_reg.weight, 0, 0.001)
            nn.init.gauss_(self.fc_reg.weight, 0, 0.001)
            nn.init.constant_(self.fc_reg.bias, 0)

    def excute(self, x):
        if self.with_avg_pool:
            x = self.avg_pool(x)
        x = x.reshape(x.shape[0], -1)
        cls_score = self.fc_cls(x) if self.with_cls else None
        bbox_pred = self.fc_reg(x) if self.with_reg else None
        return cls_score, bbox_pred

    def get_target(self, sampling_results, gt_masks, gt_labels, rcnn_train_cfg):
        pos_proposals = [res.pos_bboxes for res in sampling_results]
        neg_proposals = [res.neg_bboxes for res in sampling_results]
        pos_assigned_gt_inds = [res.pos_assigned_gt_inds  for res in sampling_results]
        pos_gt_labels = [res.pos_gt_labels for res in sampling_results]
        reg_classes = 1 if self.reg_class_agnostic else self.num_classes
        cls_reg_targets = bbox_target_rbbox(
            pos_proposals,
            neg_proposals,
            pos_assigned_gt_inds,
            gt_masks,
            pos_gt_labels,
            rcnn_train_cfg,
            reg_classes,
            target_means=self.target_means,
            target_stds=self.target_stds,
            with_module=self.with_module,
            hbb_trans=self.hbb_trans)
        return cls_reg_targets

    def get_target_rbbox(self, sampling_results, gt_bboxes, gt_labels,
                   rcnn_train_cfg):
        pos_proposals = [res.pos_bboxes for res in sampling_results]
        neg_proposals = [res.neg_bboxes for res in sampling_results]
        pos_gt_bboxes = [res.pos_gt_bboxes for res in sampling_results]
        pos_gt_labels = [res.pos_gt_labels for res in sampling_results]
        reg_classes = 1 if self.reg_class_agnostic else self.num_classes
        cls_reg_targets = rbbox_target_rbbox(
            pos_proposals,
            neg_proposals,
            pos_gt_bboxes,
            pos_gt_labels,
            rcnn_train_cfg,
            reg_classes,
            target_means=self.target_means,
            target_stds=self.target_stds)
        return cls_reg_targets

    def get_det_bboxes(self,
                       rois,
                       cls_score,
                       bbox_pred,
                       img_shape,
                       scale_factor,
                       rescale=False,
                       cfg=None):
        if isinstance(cls_score, list):
            cls_score = sum(cls_score) / float(len(cls_score))
        scores = nn.softmax(cls_score, dim=1) if cls_score is not None else None

        if rois.size(1) == 5:
            obbs = hbb2obb_v2(rois[:, 1:])
        elif rois.size(1) == 6:
            obbs = rois[:, 1:]
        else:
            print('strange size')
            import pdb
            pdb.set_trace()
        hook.record('obbs', obbs)
        hook.record('bbox_pred', bbox_pred)
        hook.record('scores', scores)
        if bbox_pred is not None:
            dbboxes = delta2dbbox_v3(obbs, bbox_pred, self.target_means,
                                    self.target_stds, img_shape)
        else:
            dbboxes = obbs

        if rescale:
            dbboxes[:, 0::5] /= scale_factor
            dbboxes[:, 1::5] /= scale_factor
            dbboxes[:, 2::5] /= scale_factor
            dbboxes[:, 3::5] /= scale_factor
        hook.record('dbboxes', dbboxes)
        hook.record('scores', scores)
        det_bboxes, det_labels = multiclass_nms_rotated(dbboxes, scores,
                                                cfg.score_thr, cfg.nms,
                                                cfg.max_per_img)
        det_bboxes = jt.contrib.concat([rotated_box_to_poly(det_bboxes), det_bboxes[:,-1:]], -1)
        hook.record('det_bboxes', det_bboxes)
        hook.record('det_labels', det_labels)
        return det_bboxes, det_labels

    def loss(self,
             cls_score,
             bbox_pred,
             labels,
             label_weights,
             bbox_targets,
             bbox_weights,
             reduce=True):
        losses = dict()
        if cls_score is not None:
            losses['rbbox_loss_cls'] = self.loss_cls(
                cls_score, labels, label_weights, reduce=reduce)
            losses['rbbox_acc'] = accuracy(cls_score, labels)
        if bbox_pred is not None:
            pos_inds = labels > 0
            if self.reg_class_agnostic:
                pos_bbox_pred = bbox_pred.view(bbox_pred.size(0), 5)[pos_inds]
            else:
                pos_bbox_pred = bbox_pred.view(bbox_pred.size(0), -1,
                                               5)[pos_inds, labels[pos_inds]]
            losses['rbbox_loss_bbox'] = self.loss_bbox(
                pos_bbox_pred,
                bbox_targets[pos_inds],
                bbox_weights[pos_inds],
                avg_factor=bbox_targets.size(0))
        return losses


# -*- coding: utf-8 -*-

import re
#import opencc
import os
import time
import csv
import json
from gensim import corpora
from utils import load_scws, cut
result_path = './result/'

AB_PATH = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'classify_dict')

cut_str = load_scws()

dictionary_3 = corpora.Dictionary.load(os.path.join(AB_PATH, 'new_classify.dict'))
step3_score = {}
with open(os.path.join(AB_PATH, 'new_classify.txt')) as f:
    for l in f:
        lis = l.rstrip().split()
        step3_score[int(lis[0])] = [float(lis[1]), float(lis[2]), float(lis[3]), float(lis[4]), float(lis[5])]

def triple_classifier(text):

    entries = cut(cut_str, text)
    entry = [e.decode('utf-8', 'ignore') for e in entries]
    keywords_list = entry

    bow = dictionary_3.doc2bow(entry)
    s = [1, 1, 1, 1, 1]
    for pair in bow:
        s[0] = s[0] * (step3_score[pair[0]][0] ** pair[1])
        s[1] = s[1] * (step3_score[pair[0]][1] ** pair[1])
        s[2] = s[2] * (step3_score[pair[0]][2] ** pair[1])
        s[3] = s[3] * (step3_score[pair[0]][3] ** pair[1])
        s[4] = s[4] * (step3_score[pair[0]][4] ** pair[1])

    s_max = max(s)
    s_dis = abs(s[0]-s_max) + abs(s[1]-s_max) + abs(s[2]-s_max) + abs(s[3]-s_max) + abs(s[4]-s_max)
    if (s_dis/5)/s_max <= 0.7:
        label = 'other'
    else:

        if s[0] > s[1] and s[0] > s[2] and s[0] > s[3] and s[0] > s[4]:#政治领域
            label = 'politics'
        elif s[1] > s[0] and s[1] > s[2] and s[1] > s[3] and s[1] > s[4]:#外交领域
            label = 'diplomacy'
        elif s[2] > s[0] and s[2] > s[1] and s[2] > s[3] and s[2] > s[4]:#经济领域
            label = 'economic'
        elif s[3] > s[0] and s[3] > s[1] and s[3] > s[2] and s[3] > s[4]:#暴恐领域
            label = 'violence'
        elif s[4] > s[0] and s[4] > s[1] and s[4] > s[3] and s[4] > s[2]:#政变领域
            label = 'coup'
        else:
            label = 'other'

    return label

def get_file_data(_id):

    f = open(result_path + 'recommend_news_%s.jl' % _id)
    news_data = []
    for line in f:
        news_dict = json.loads(line)
        news_data.append(news_dict)

    return news_data

def write_file_data(_id,result):

    f = open(result_path + 'classify_result_%s.jl' % _id,'a')

    for item in result:
        row = json.dumps(item)
        f.write(row+'\n')

    f.close()

if __name__ == '__main__':

    news_data = get_file_data('1487665506')
    result = []
    for news in news_data:
        if news.has_key('title') and news.has_key('summary'):
            text = news['title'].encode('utf-8','ignore') + news['summary'].encode('utf-8','ignore')
        elif news.has_key('title') and not news.has_key('summary'):
            text = news['title'].encode('utf-8','ignore')
        elif not news.has_key('title') and news.has_key('summary'):
            text = news['summary'].encode('utf-8','ignore')
        else:
            text = ''

        if len(text):#非空字符
            label = triple_classifier(text)
        else:
            label = 'other'
        news['label'] = label
        result.append(news)

    write_file_data('1487665506',result)


import sys
import re

def istitle(wordslist):
   return wordslist[0][-1] != "." and wordslist[1][0].isupper() and wordslist[2][0].isupper()

# Remove empty fields in a list
def remove_spaces(lines):
    return list(filter(lambda a: a != "", lines))

# Returns True if the text has neither English nor Chinese characters
def find_chinese_and_english(line):
    regex = "[a-zA-z\u4e00-\u9fff]"
    return re.findall(regex, line) == []

# Check if a word uses only the English alphabets
def is_english(word):
    try:
        return word.encode('ascii').isalpha()
    except:
        return False

# Build dictionary of Chinese words that should be merged back
def build_dict(file):
    _dict = {}
    for _ in load_file(file):
        if _[-2] in _dict:
            _dict[_[-2]].add(_[0])
        else:
            _dict[_[-2]] = set(_[0])
    return _dict

# Remove spaces between Chinese words
def chinese_smush(line):
    characters = [""] + list(line) + [""]
    for index in range(len(characters)):
        if characters[index].isspace() and not is_english(characters[index-1]) and not is_english(characters[index+1]):
            characters[index] = ""
    return "".join(remove_spaces(characters))

# Check common word pairing which gets broken up accidentally
def check_word_pair(dict, prev_line, curr_line):
    curr_first_char, prev_last_char = curr_line[0], prev_line[-1]
    prev_characters = list(prev_line)
    curr_characters = list(curr_line)
    return curr_first_char in dict and prev_last_char in dict[curr_first_char] \
    or curr_characters[0] == "」" or curr_characters[0] == "）" \
    or prev_characters[-1] == "「" or prev_characters[-1] == "（" \

# Check exceptions to merge back to the previous sentence based on the last words of the previous line
# Backward Pass
def backward_pass(start_exceptions, prev_line, curr_line):
    first_char, last_char = curr_line[0], curr_line[-1]
    curr_split, prev_split = curr_line.split(), prev_line.split()

    curr_first_word = curr_split[0]
    curr_first_word_last_char = curr_first_word[-1]
    prev_last_word = prev_split[-1]
    prev_last_char = prev_line[-1]

    # Bunch of if/else return statements
    return (last_char == "." or last_char == ":" or last_char == ";") and first_char.islower() \
    or prev_last_word in start_exceptions \
    or (len(curr_split) == 1) and (curr_line.islower() or curr_first_word_last_char == ")" or curr_first_word_last_char == ".") \
    # or (len(prev_split) == 1) and (prev_last_char == ".")
    # or ")" in curr_first_word \

# Get exception files
def get_exceptions(start, end):
    print ("Getting exceptions from exception files...")
    with open(start, "r", encoding="utf-8") as file:
        start_exceptions = file.read().split("\n")
    with open(end, "r", encoding="utf-8") as file:
        end_exceptions = file.read().split("\n")
    return start_exceptions, end_exceptions

# Load file to be processed
def load_file(file):
    print ("Loading", file,"to be processed...")
    with open(file, "r", encoding="utf-8") as inp:
        return inp.readlines()

# Write output to file
def write_output(lines, file):
    print ("Writing to file...")
    #lines = [line[1:].replace("  ", " ") + "\n" if line[0].isspace() else line.replace("  ", " ") + "\n" for line in lines]
    lines = [line.replace("  ", " ") + "\n" for line in lines]
    lines = purge_noise(lines)
    with open(file, "w", encoding="utf-8") as outp:
        outp.writelines(lines)
        print ("Completed.")

# Splitting handler
def split_by(data, symbol):
    output = []
    data = list(filter(None, data))

    if symbol == "\n":
        print ("Splitting data by line break")
        for _ in data:
            output.append(_.replace("\n", ""))

    elif symbol == ". ":
        print ("Splitting data by full stop")
        for _ in data:
            if symbol in _:
                split = _.split(symbol)
                mylist = [split[i] + "." for i in range(len(split)-1)] + [split[-1]]

                for segment in mylist:
                    output.append(segment)

            else:
                output.append(_)
    else:
        print ("Splitting data by ", symbol)
        for _ in data:
            if symbol in _:
                split = _.split(symbol)
                mylist = [split[i] + symbol for i in range(len(split)-1)] + [split[-1]]
                for segment in mylist:
                    output.append(segment)
            else:
                output.append(_)

    return output

# Remove noises
def purge_noise(lines):
    noise_file = "noise.txt"
    noises = load_file(noise_file)
    output = []

    for line in lines:
        if find_chinese_and_english(line):
            print ("Purged: ", line)
            continue
        for noise in noises:
            if line == noise or find_chinese_and_english(line.replace(noise, "")):
                print ("Purged: ", line)
                break
        else:
            output.append(line)
    return output

if __name__ == "__main__":
    input_file, output_file, language = sys.argv[1], sys.argv[2], sys.argv[3]
    se, ee = "start_exceptions.txt", "end_exceptions.txt"
    zh_dict_file = "zh_dict.txt"

    print ("Processing file in the following language: ", language)

    # Splitting part
    symbols = {
        "en" : {". ", ";", ":", "\n"},
        "zh" : {"。", "：", "；", "\n"}
    }

    start_exceptions, end_exceptions = get_exceptions(se, ee)
    data = load_file(input_file)
    lines = data

    for symbol in symbols[language]:
        lines = split_by(lines, symbol)
        lines = remove_spaces(lines)

        # print (lines)
    if language == "en":
        for index in range(len(lines)-1, 1, -1):
            if backward_pass(start_exceptions, lines[index-1], lines[index]):
                lines[index-1] = " ".join([lines[index-1], lines[index]])
                lines[index] = ""

    elif language == "zh":
        zh_dict = build_dict(zh_dict_file)

        # Remove spaces between characters
        for index in range(len(lines)):
            lines[index] = chinese_smush(lines[index])

        for index in range(len(lines)-1, 1, -1):
            if check_word_pair(zh_dict, lines[index-1], lines[index]):
                lines[index-1] = "".join([lines[index-1], lines[index]])
                lines[index] = ""

    write_output(lines, output_file)


import os
import numpy as np
import cv2

from preprocess import prepareTrain, prepareTest

from config import conf

def training_generator(data, angle_bias=1.0, batch_size=128):
    batch_X = np.zeros((batch_size, conf.row, conf.col, 3))
    batch_y = np.zeros(batch_size)
    data_folder = conf.data_folder
    print (angle_bias, len(data))
    cnt = 0
    while 1:
        while (cnt < batch_size):
            # Randomly select an image.
            index = np.random.randint(len(data))
            camera = ['center', 'left', 'right']
            ci = np.random.randint(3)

            file = os.path.join(data_folder, data[camera[ci]].iloc[index].strip())

            #shift angle for center, left, and right camera
            shift = [0.0, conf.left_offset, conf.right_offset]

            image = cv2.imread(file)

            y = data['steering'].iloc[index] + shift[ci]

            # Apply random brightness,
            X, y = prepareTrain(image, y)

            # A higher bias value will exclude smaller (< 0.1 absolute value) angles.
            threshold = np.random.uniform()
            if abs(y) < 0.1:
                if angle_bias > threshold:
                    continue

            batch_X[cnt] = X
            batch_y[cnt] = y

            cnt += 1

        yield (batch_X, batch_y)

def validation_generator(data, batch_size=128):
    batch_X = np.zeros((batch_size, conf.row, conf.col, 3))
    batch_y = np.zeros(batch_size)
    data_folder = conf.data_folder
    cnt = 0
    while 1:
        while (cnt < batch_size):
            # Randomly select an image.
            index = np.random.randint(len(data))

            file = os.path.join(data_folder, data['center'].iloc[index].strip())

            image = cv2.imread(file)

            y = data['steering'].iloc[index]

            X = prepareTest(image)

            batch_X[cnt] = X
            batch_y[cnt] = y

            cnt += 1

        yield (batch_X, batch_y)


import numpy as np
import pandas as pd
from sklearn.preprocessing import MinMaxScaler
import torch
from torch.utils.data import Dataset

class YahooDataset(Dataset):
    def prepare_dataset(self, dataset_path):
        dataset = pd.read_csv(dataset_path)
        dataset = dataset[['Close']]
        class_idx = 0
        return dataset, class_idx

    def split_dataset(self, dataset, train, train_rate):
        n = len(dataset)
        if train:
            return dataset[:int(n * train_rate)]
        else:
            return dataset[int(n * train_rate):]

    def __init__(self, dataset_path, window_len, forecast_len=1, train=True, train_rate=0.7, scalertype="none", scaler=None):
        super().__init__()

        if train == True and (scalertype != "none" and scalertype != "minmax"):
            raise Exception("scaler not recognized, use 'none' or 'minmax'")

        dataset, class_idx = self.prepare_dataset(dataset_path)
        self.class_idx = class_idx
        self.forecast_len = forecast_len
        self.dataset = self.split_dataset(dataset, train, train_rate).to_numpy()
        self.window_len = window_len
        self.scaler = scaler
        if train == True and scalertype == "minmax":
            self.scaler = MinMaxScaler()
            self.scaler.fit(self.dataset)

    def __len__(self):
        return len(self.dataset) - self.window_len - 2 * self.forecast_len + 1

    def get_scaler(self):
        return self.scaler

    def __getitem__(self, index):
        input = self.dataset[index:index + self.window_len + 2 * self.forecast_len - 1, :]
        if self.scaler is not None:
            input = self.scaler.transform(input)
        return torch.from_numpy(input).to(torch.float32), self.window_len, self.class_idx


#!/usr/bin/env python3
"""
Firmware Debugger
"""
import sys
import time
import subprocess
import importlib
import traceback
import argparse

try:
  import serial
  import numpy as np
  import pyqtgraph as pg
  from PyQt5.QtCore import QTimer
  import matplotlib.pylab as plt
except ImportError:
  print(traceback.format_exc())

def install_package(package):
  """ Install package """
  if importlib.util.find_spec(package) is None:
    subprocess.check_call([sys.executable, "-m", "pip", "install", package])

def parse_serial_data_line(line):
  """ Parse serial data """
  data = {}

  line = line.strip()
  if len(line) == 0:
    return data

  if type(line) is str:
    for el in line.split(" "):
      key = el.split(":")[0].strip()
      val = float(el.split(":")[1].strip())
      data[key] = val
  else:
    for el in line.split(b" "):
      key = el.split(b":")[0].strip().decode("ascii")
      val = float(el.split(b":")[1].strip())
      data[key] = val

  data['ts'] = data['ts'] * 1e-6

  return data

def record_telemetry_data(save_path, record_duration):
  """ Record Telemetry Data """
  # Connect to FCU
  s = serial.Serial()
  s.port = '/dev/ttyACM0'
  s.baudrate = 921600
  s.timeout = 10
  s.open()

  if s.is_open is True:
    print("Connected to FCU ...")
  else:
    print("Failed to connect to FCU ...")
    sys.exit(-1)

  # Record
  line = s.readline()
  if line is None:
    return

  data = []
  time_start = time.time()
  while line:
    data.append(line.decode("ascii"))
    line = s.readline()
    if (time.time() - time_start) > record_duration:
      break

  # Save data
  log_file = open(save_path, "w")
  for data_line in data:
    log_file.write(data_line)

def analyze_timestamps():
  """ Analyze Timestamps """
  log_file = "/tmp/debugger.log"
  record_duration = 10.0

  # Record telemetry data
  record_telemetry_data(log_file, record_duration)

  # Parse log file
  timestamps = []
  for line in open(log_file, "r"):
    data = parse_serial_data_line(line)
    timestamps.append(data['ts'])
  timestamps = np.array(timestamps)

  # Calculate rates
  rates = []
  start_idx = 100
  for k in range(start_idx, len(timestamps)):
    ts_k = timestamps[k]
    ts_km1 = timestamps[k - 1]
    dt = ts_k - ts_km1
    rates.append(1.0 / dt)

  # Plot
  plt.plot(timestamps[start_idx:], rates, 'ro')
  plt.ylim([0, 1000])
  plt.xlabel("Time [s]")
  plt.ylabel("Rate [Hz]")
  plt.show()

class RTLinePlot:
  """ Real-Time Line Plot """
  def __init__(self, win, title, x_key, y_keys, x_label, y_label, **kwargs):
    self.plot = win.addPlot(title=title)
    self.plot.addLegend()
    self.plot.setLabel("bottom", x_label)
    self.plot.setLabel("left", y_label)
    self.plot.setDownsampling(mode='peak')
    self.plot.setClipToView(True)

    self.y_min = kwargs.get("y_min")
    self.y_max = kwargs.get("y_max")
    self.y_padding = kwargs.get("y_padding", 1)
    if self.y_min and self.y_max:
      # self.plot.setLimits(yMin=self.y_min, yMax=self.y_max)
      self.plot.setYRange(self.y_min, self.y_max)

    self.win_size = kwargs.get("win_size", 1000)
    self.colors = kwargs.get("colors", ["r", "g", "b", "c", "y", "m"])
    self.x_key = x_key
    self.y_keys = y_keys
    self.data = {}
    self.curves = {}

    self.data[x_key] = []
    for i, y_key in enumerate(self.y_keys):
      self.data[y_key] = []
      self.curves[y_key] = self.plot.plot(pen=self.colors[i],
                                          name=y_key,
                                          skipFiniteCheck=True)

  def set_xlink(self, rt_plot):
    """ Set xlink """
    self.plot.setXLink(rt_plot.plot)

  def update_data(self, data):
    """ Update Data """
    self.data[self.x_key].append(data[self.x_key])
    for y_key in self.y_keys:
      if y_key in data:
        self.data[y_key].append(data[y_key])

  def update_plots(self):
    """ Update Plots """
    for y_key in self.y_keys:
      x_window = self.data[self.x_key][-self.win_size:]
      y_window = self.data[y_key][-self.win_size:]
      self.curves[y_key].setData(x_window, y_window)
      if self.y_min and self.y_max:
        # self.plot.setLimits(yMin=self.y_min, yMax=self.y_max)
        self.plot.setYRange(self.y_min, self.y_max)

class FirmwareDebugger:
  """ Firmware Debugger """
  def __init__(self, **kwargs):
    self.log_file = kwargs.get("log_file")
    self.offline_mode = (self.log_file is not None)
    self.win_size = -1 if self.offline_mode else 1000
    self.s = None  # Serial
    self.app = None
    self.win = None
    self.timestamps = []
    self.last_plotted = None

    self._setup_gui()
    if self.log_file:
      self._load_log_data(self.log_file)
    else:
      self.log = open("/tmp/debugger.log", "w")
      self._setup_uart()
      self._start_plots()

  def _setup_uart(self):
    """ Setup UART comms """
    # Setup serial communication
    self.serial = serial.Serial()
    self.serial.port = '/dev/ttyACM0'
    self.serial.baudrate = 921600
    self.serial.timeout = 0
    self.serial.open()

    if self.serial.is_open is True:
      print("Connected to FCU ...")
    else:
      print("Failed to connect to FCU ...")
      sys.exit(-1)

  def _setup_gui(self):
    """ Setup GUI """
    self.app = pg.mkQApp("Firmware Debugger")
    self.win = pg.GraphicsLayoutWidget(show=True, title="Firmware Debugger")
    self.win.resize(1910, 768)
    pg.setConfigOptions(antialias=True)

    title = "Gyroscope"
    x_key = "ts"
    y_keys = ["gyro_x", "gyro_y", "gyro_z"]
    x_label = "Time [s]"
    y_label = "Angular Velocity [rad / s]"
    kwargs = {"y_min": -5.0, "y_max": 5.0, "win_size": self.win_size}
    self.accel_plot = RTLinePlot(
        self.win,
        title,
        x_key,
        y_keys,
        x_label,
        y_label,
        **kwargs,
    )

    title = "Accelerometer"
    x_key = "ts"
    y_keys = ["accel_x", "accel_y", "accel_z"]
    x_label = "Time [s]"
    y_label = "Acceleration [m / s^2]"
    kwargs = {"y_min": -12.0, "y_max": 12.0, "win_size": self.win_size}
    self.gyro_plot = RTLinePlot(
        self.win,
        title,
        x_key,
        y_keys,
        x_label,
        y_label,
        **kwargs,
    )
    self.gyro_plot.set_xlink(self.accel_plot)

    title = "Attitude"
    x_key = "ts"
    y_keys = [
        "roll",
        "pitch",
        "yaw",
        "roll_setpoint",
        "pitch_setpoint",
        "yaw_setpoint",
    ]
    x_label = "Time [s]"
    y_label = "Attitude [deg]"
    kwargs = {"y_min": -60.0, "y_max": 60.0, "win_size": self.win_size}
    self.attitude_plot = RTLinePlot(
        self.win,
        title,
        x_key,
        y_keys,
        x_label,
        y_label,
        **kwargs,
    )
    self.attitude_plot.set_xlink(self.gyro_plot)

    self.win.nextRow()
    title = "SBUS"
    x_key = "ts"
    y_keys = ["ch[0]", "ch[1]", "ch[2]", "ch[3]", "ch[4]", "ch[5]"]
    x_label = "Time [s]"
    y_label = "SBUS Value"
    kwargs = {"y_min": 175, "y_max": 1850, "win_size": self.win_size}
    self.sbus_plot = RTLinePlot(
        self.win,
        title,
        x_key,
        y_keys,
        x_label,
        y_label,
        **kwargs,
    )
    self.sbus_plot.set_xlink(self.attitude_plot)

    title = "Motor Outputs"
    x_key = "ts"
    y_keys = ["outputs[0]", "outputs[1]", "outputs[2]", "outputs[3]"]
    x_label = "Time [s]"
    y_label = "Motor Thrust [%]"
    kwargs = {"y_min": 0.0, "y_max": 1.0, "win_size": self.win_size}
    self.motors_plot = RTLinePlot(
        self.win,
        title,
        x_key,
        y_keys,
        x_label,
        y_label,
        **kwargs,
    )
    self.motors_plot.set_xlink(self.attitude_plot)

    title = "Height"
    x_key = "ts"
    y_keys = ["height"]
    x_label = "Time [s]"
    y_label = "Height [m]"
    kwargs = {"win_size": self.win_size}
    self.height_plot = RTLinePlot(
        self.win,
        title,
        x_key,
        y_keys,
        x_label,
        y_label,
        **kwargs,
    )
    self.height_plot.set_xlink(self.motors_plot)

  def _start_plots(self):
    """ Start plots """
    timer = QTimer()
    timer.timeout.connect(self._update_plots)
    timer.start()
    pg.exec()

  def _load_log_data(self, log_file):
    """ Load log data """
    # Load log data to plots
    for line in open(log_file, "r"):
      data = parse_serial_data_line(line)
      self.timestamps.append(data['ts'])

      self.sbus_plot.update_data(data)
      self.gyro_plot.update_data(data)
      self.accel_plot.update_data(data)
      self.attitude_plot.update_data(data)
      self.motors_plot.update_data(data)
      self.height_plot.update_data(data)

    # Update plots
    self.sbus_plot.update_plots()
    self.gyro_plot.update_plots()
    self.accel_plot.update_plots()
    self.attitude_plot.update_plots()
    self.motors_plot.update_plots()
    self.height_plot.update_plots()
    self.last_plotted = time.time()

    # Run GUI
    pg.exec()

  def _update_plots(self):
    """ Update plots """
    # Read serial buffer
    line = self.serial.readline()
    if line is None:
      return

    while line:
      data = parse_serial_data_line(line)
      self.timestamps.append(data['ts'])
      self.sbus_plot.update_data(data)
      self.gyro_plot.update_data(data)
      self.accel_plot.update_data(data)
      self.attitude_plot.update_data(data)
      self.motors_plot.update_data(data)
      self.height_plot.update_data(data)
      self.log.write(line.decode("ascii"))
      line = self.serial.readline()

    # Update plots
    time_now = time.time()
    if self.last_plotted is None or (time_now - self.last_plotted) > 0.05:
      self.sbus_plot.update_plots()
      self.gyro_plot.update_plots()
      self.accel_plot.update_plots()
      self.attitude_plot.update_plots()
      self.motors_plot.update_plots()
      self.height_plot.update_plots()
      self.last_plotted = time.time()

if __name__ == "__main__":
  install_package("pyserial")
  install_package("numpy")
  install_package("pyqtgraph")

  # log_file = "/tmp/debugger.log"
  # record_duration = 5.0
  # record_telemetry_data(log_file, record_duration)
  # FirmwareDebugger(log_file=log_file)

  # analyze_timestamps()

  parser = argparse.ArgumentParser()
  parser.add_argument("--log_file", nargs="?", default=None)
  args = parser.parse_args()
  FirmwareDebugger(log_file=args.log_file)