diff --git a/alembic/versions/ab038e16ec9a_standardize_columns.py b/alembic/versions/ab038e16ec9a_standardize_columns.py new file mode 100644 index 0000000..fa1b888 --- /dev/null +++ b/alembic/versions/ab038e16ec9a_standardize_columns.py @@ -0,0 +1,28 @@ +"""standardize_columns + +Revision ID: ab038e16ec9a +Revises: 4b8ce290f890 +Create Date: 2016-06-28 15:44:52.141256 + +""" + +# revision identifiers, used by Alembic. +revision = 'ab038e16ec9a' +down_revision = '4b8ce290f890' +branch_labels = None +depends_on = None + +from alembic import op +import sqlalchemy as sa + + +def upgrade(): + op.add_column('ofxaccount', sa.Column('deleted', sa.DateTime(), nullable=True)) + op.add_column('ofxrecord', sa.Column('deleted', sa.DateTime(), nullable=True)) + op.add_column('ofxsource', sa.Column('deleted', sa.DateTime(), nullable=True)) + + +def downgrade(): + op.drop_column('ofxsource', 'deleted') + op.drop_column('ofxrecord', 'deleted') + op.drop_column('ofxaccount', 'deleted') diff --git a/templates/accounts.html b/templates/accounts.html index c719ca9..1f7fd15 100644 --- a/templates/accounts.html +++ b/templates/accounts.html @@ -3,12 +3,19 @@

Accounts

{% if accounts %} - + {% for account in accounts %} - + + + {% endfor %}
NameTypeInstitution
NameTypeInstitutionLast Update
{{ account.name }} {{ account.type }}{{ account.institution }}{{ account.source.name }}Never +
+ + +
+
diff --git a/tests/files/query_transactions.ofx b/tests/files/query_transactions.ofx new file mode 100644 index 0000000..83a9ed9 --- /dev/null +++ b/tests/files/query_transactions.ofx @@ -0,0 +1,42 @@ +OFXHEADER:100 +DATA:OFXSGML +VERSION:102 +SECURITY:NONE +ENCODING:USASCII +CHARSET:1252 +COMPRESSION:NONE +OLDFILEUID:NONE +NEWFILEUID:NONE + + + + +20160102030405.000[-7:MST] +123456789 +1234 +ENG + +AFCU +12345 + +QWIN +1200 + + + + +00000000 + + +1234567 +123456-0.9:CHK +CHECKING + + +20160102 +Y + + + + + diff --git a/vanth/celery.py b/vanth/celery.py new file mode 100644 index 0000000..468a140 --- /dev/null +++ b/vanth/celery.py @@ -0,0 +1,19 @@ +import celery + +import vanth.download +import vanth.main +import vanth.platform.ofxaccount +import vanth.platform.ofxrecord +import vanth.platform.ofxsource + +app = celery.Celery('vanth') +app.conf.CELERY_ACCEPT_CONTENT = ['json', 'msgpack', 'yaml'] +app.conf.CELERY_TASK_SERIALIZER = 'json' +app.conf.CELERY_ALWAYS_EAGER = True + +@app.task() +def update_account(account_uuid): + account = vanth.platform.ofxaccount.by_uuid(account_uuid)[0] + source = vanth.platform.ofxsource.by_uuid(account['source']['uuid'])[0] + document = vanth.download.transactions(source, account) + vanth.platform.ofxrecord.ensure_exists(account, document.body.statement.transactions.items) diff --git a/vanth/download.py b/vanth/download.py new file mode 100644 index 0000000..a9f0daf --- /dev/null +++ b/vanth/download.py @@ -0,0 +1,17 @@ +import requests + +import vanth.ofx +import vanth.platform.ofxaccount + + +def do_all(): + sources = {source['name']: source for source in vanth.platform.ofxsource.get()} + accounts = vanth.platform.ofxaccount.by_user(user_id=None) + for account in accounts: + transactions(sources[account['institution']], account) + +def transactions(source, account): + body = vanth.ofx.query_transactions(source, account) + response = requests.post('https://ofx.americafirst.com/', data=body, headers={'Content-Type': 'application/x-ofx'}) + assert response.ok, response.text + return vanth.ofx.parse(response.text) diff --git a/vanth/pages/accounts.py b/vanth/pages/accounts.py index 96a2935..9b7f61d 100644 --- a/vanth/pages/accounts.py +++ b/vanth/pages/accounts.py @@ -1,5 +1,7 @@ import flask +import vanth.celery +import vanth.pages.tools import vanth.platform.ofxaccount import vanth.platform.ofxsource @@ -7,27 +9,28 @@ blueprint = flask.Blueprint('accounts', __name__) @blueprint.route('/accounts/', methods=['GET']) def get_accounts(): - my_accounts = vanth.platform.ofxaccount.get(flask.session['user_id']) + my_accounts = vanth.platform.ofxaccount.by_user(flask.session['user_id']) sources = vanth.platform.ofxsource.get() return flask.render_template('accounts.html', accounts=my_accounts, sources=sources) @blueprint.route('/account/', methods=['POST']) -def post_account(): - account_id = flask.request.form.get('account_id') - account_type = flask.request.form.get('account_type') - institution = flask.request.form.get('institution') - name = flask.request.form.get('name') - password = flask.request.form.get('password') - user_id = flask.request.form.get('user_id') - - - vanth.platform.ofxaccount.create({ - 'owner' : flask.session['user_id'], - 'account_id' : account_id, - 'institution' : institution, - 'name' : name, - 'password' : password, - 'type' : account_type, - 'user_id' : user_id, - }) +@vanth.pages.tools.parse({ + 'account_id' : str, + 'account_type' : str, + 'institution' : str, + 'name' : str, + 'password' : str, + 'user_id' : str, +}) +def post_account(arguments): + arguments['owner'] = flask.session['user_id'] + vanth.platform.ofxaccount.create(arguments) + return flask.redirect('/accounts/') + +@blueprint.route('/update/', methods=['POST']) +@vanth.pages.tools.parse({ + 'account_uuid' : str, +}) +def post_update(arguments): + vanth.celery.update_account.delay(**arguments) return flask.redirect('/accounts/') diff --git a/vanth/platform/ofxaccount.py b/vanth/platform/ofxaccount.py index 1c7b69f..f931939 100644 --- a/vanth/platform/ofxaccount.py +++ b/vanth/platform/ofxaccount.py @@ -7,32 +7,49 @@ import vanth.platform.ofxsource import vanth.tables -def get(user_id): - engine = chryso.connection.get() - query = sqlalchemy.select([ - vanth.tables.OFXSource.c.name.label('institution'), +def _select(): + return sqlalchemy.select([ + vanth.tables.OFXAccount.c.account_id, vanth.tables.OFXAccount.c.name, + vanth.tables.OFXAccount.c.password, vanth.tables.OFXAccount.c.source, vanth.tables.OFXAccount.c.type, vanth.tables.OFXAccount.c.user_id, vanth.tables.OFXAccount.c.uuid, + vanth.tables.OFXSource.c.name.label('source.name'), + vanth.tables.OFXSource.c.uuid.label('source.uuid'), ]).where( vanth.tables.OFXAccount.c.source == vanth.tables.OFXSource.c.uuid ) - if user_id: - query = query.where( - vanth.tables.OFXAccount.c.owner == user_id - ) + +def _execute_and_convert(query): + engine = chryso.connection.get() results = engine.execute(query) return [{ - 'institution' : result[vanth.tables.OFXSource.c.name.label('institution')], + 'account_id' : result[vanth.tables.OFXAccount.c.account_id], 'name' : result[vanth.tables.OFXAccount.c.name], - 'source' : result[vanth.tables.OFXAccount.c.source], + 'password' : result[vanth.tables.OFXAccount.c.password], + 'source' : { + 'name' : result[vanth.tables.OFXSource.c.name.label('source.name')], + 'uuid' : result[vanth.tables.OFXSource.c.name.label('source.uuid')], + }, 'type' : result[vanth.tables.OFXAccount.c.type], 'user_id' : result[vanth.tables.OFXAccount.c.user_id], 'uuid' : result[vanth.tables.OFXAccount.c.uuid], } for result in results] +def by_uuid(account_uuid): + query = _select().where(vanth.tables.OFXAccount.c.uuid == account_uuid) + return _execute_and_convert(query) + +def by_user(user_id): + query = _select() + if user_id: + query = query.where( + vanth.tables.OFXAccount.c.owner == user_id + ) + return _execute_and_convert(query) + def create(values): engine = chryso.connection.get() diff --git a/vanth/platform/ofxrecord.py b/vanth/platform/ofxrecord.py new file mode 100644 index 0000000..4369684 --- /dev/null +++ b/vanth/platform/ofxrecord.py @@ -0,0 +1,34 @@ +import logging +import uuid + +import chryso.connection +import sqlalchemy + +import vanth.tables + +LOGGER = logging.getLogger(__name__) + +def ensure_exists(account, transactions): + engine = chryso.connection.get() + query = sqlalchemy.select([ + vanth.tables.OFXRecord.c.fid, + ]).where(vanth.tables.OFXRecord.c.ofxaccount == account['uuid']) + results = engine.execute(query).fetchall() + LOGGER.debug("Found %d OFX records for %s", len(results), account) + known_records = {result[vanth.tables.OFXRecord.c.fid] for result in results} + new_records = [transaction for transaction in transactions if transaction.id not in known_records] + LOGGER.debug("Have %d new transactions to save", len(new_records)) + to_insert = [{ + 'amount' : transaction.amount, + 'available' : transaction.available, + 'fid' : transaction.id, + 'memo' : transaction.memo, + 'name' : transaction.name, + 'ofxaccount' : account['uuid'], + 'posted' : transaction.posted, + 'type' : transaction.type, + 'uuid' : uuid.uuid4(), + } for transaction in new_records] + if to_insert: + engine.execute(vanth.tables.OFXRecord.insert(), to_insert) # pylint: disable=no-value-for-parameter + LOGGER.debug("Done inserting %d records", len(new_records)) diff --git a/vanth/platform/ofxsource.py b/vanth/platform/ofxsource.py index 764745f..3d577ca 100644 --- a/vanth/platform/ofxsource.py +++ b/vanth/platform/ofxsource.py @@ -6,8 +6,15 @@ import vanth.tables LOGGER = logging.getLogger(__name__) -def get(): +def _query_and_convert(query): engine = chryso.connection.get() - query = vanth.tables.OFXSource.select() results = engine.execute(query).fetchall() return [dict(result) for result in results] + +def by_uuid(uuid): + query = vanth.tables.OFXSource.select().where(vanth.tables.OFXSource.c.uuid == uuid) + return _query_and_convert(query) + +def get(): + query = vanth.tables.OFXSource.select() + return _query_and_convert(query) diff --git a/vanth/tables.py b/vanth/tables.py index 34db0ca..e126946 100644 --- a/vanth/tables.py +++ b/vanth/tables.py @@ -1,10 +1,22 @@ import chryso.constants from sqlalchemy import (Column, Date, DateTime, Float, ForeignKey, Integer, - MetaData, String, Table, UniqueConstraint, func) + MetaData, String, Table, UniqueConstraint, func, text) from sqlalchemy.dialects.postgresql import UUID metadata = MetaData(naming_convention=chryso.constants.CONVENTION) +def table(name, *args, **kwargs): + return Table( + name, + metadata, + Column('uuid', UUID(as_uuid=True), primary_key=True, server_default=text('gen_random_uuid()')), + Column('created', DateTime, server_default=func.now(), nullable=False), + Column('updated', DateTime, server_default=func.now(), onupdate=func.now(), nullable=False), + Column('deleted', DateTime, nullable=True), + *args, + **kwargs + ) + User = Table('users', metadata, Column('uuid', UUID(), primary_key=True), Column('username', String(255), nullable=False), @@ -17,8 +29,7 @@ User = Table('users', metadata, UniqueConstraint('username', name='uq_user_username'), ) -CreditCard = Table('credit_card', metadata, - Column('uuid', UUID(as_uuid=True), primary_key=True), +CreditCard = table('credit_card', Column('brand', String(20), nullable=False), # The brand of the card, like 'visa' Column('card_id', String(100), nullable=False), # The ID of the card from Stripe Column('country', String(1024), nullable=False), # The Country of the card, like 'US' @@ -28,23 +39,18 @@ CreditCard = Table('credit_card', metadata, Column('last_four', Integer(), nullable=False), # The last four digits of the card Column('token', String(), nullable=False), # The token we can use with Stripe to do stuff Column('user_uri', String(2048), nullable=False), # The URI of the user that created the record - Column('created', DateTime(), nullable=False, server_default=func.now()), - Column('updated', DateTime(), nullable=False, server_default=func.now(), onupdate=func.now()), - Column('deleted', DateTime(), nullable=True), UniqueConstraint('card_id', name='uq_credit_card_id'), ) -OFXSource = Table('ofxsource', metadata, - Column('uuid', UUID(as_uuid=True), primary_key=True), +OFXSource = table('ofxsource', Column('name', String(255), nullable=False), # The name of the institution such as 'America First Credit Union' Column('fid', String(255), nullable=False), # The FID of the institution, such as 54324 - Column('bankid', String(255), nullable=False), # The bank ID of the institution such as 324377516. This may be a routing number - Column('created', DateTime(), nullable=False, server_default=func.now()), - Column('updated', DateTime(), nullable=False, server_default=func.now(), onupdate=func.now()), + Column('bankid', String(255), nullable=False), # The bank ID of the institution such as 324377516. + # This may be a routing number UniqueConstraint('fid', name='uq_ofxsource_fid'), ) -OFXAccount = Table('ofxaccount', metadata, +OFXAccount = table('ofxaccount', Column('account_id', String(255), nullable=False), # 123456-0.9:CHK Column('name', String(255), nullable=False), # My checking account Column('owner', None, ForeignKey(User.c.uuid, name='fk_user'), nullable=False), @@ -54,12 +60,9 @@ OFXAccount = Table('ofxaccount', metadata, Column('user_id', String(255), nullable=False), # The user ID for the bank Column('uuid', UUID(as_uuid=True), primary_key=True), - Column('created', DateTime(), nullable=False, server_default=func.now()), - Column('updated', DateTime(), nullable=False, server_default=func.now(), onupdate=func.now()), ) -OFXRecord = Table('ofxrecord', metadata, - Column('uuid', UUID(as_uuid=True), primary_key=True), +OFXRecord = table('ofxrecord', Column('fid', String(255), nullable=False), # The Financial institution's ID Column('amount', Float(), nullable=False), # The amount of the record, like -177.91 Column('available', Date(), nullable=True), # The date the record was available @@ -68,6 +71,4 @@ OFXRecord = Table('ofxrecord', metadata, Column('memo', String(2048), nullable=True), # The memo of the transaction, like 'POINT OF SALE PURCHASE #0005727' Column('type', String(255), nullable=True), # The type of the record, like 'POS' Column('ofxaccount', None, ForeignKey(OFXAccount.c.uuid, name='fk_ofxaccount'), nullable=False), - Column('created', DateTime(), nullable=False, server_default=func.now()), - Column('updated', DateTime(), nullable=False, server_default=func.now(), onupdate=func.now()), )