eliribble
/
vanth
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Standardize columns in our table 

Like when we last did an update and whether or not something is deleted.
Nice for cleanup and code reuse
This commit is contained in:
Eli Ribble 2016-06-28 15:46:18 -06:00
parent 2ebdd6f99e
commit f536f21d3c
10 changed files with 226 additions and 51 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

28
alembic/versions/ab038e16ec9a_standardize_columns.py Normal file
View File

@ -0,0 +1,28 @@
"""standardize_columns
Revision ID: ab038e16ec9a
Revises: 4b8ce290f890
Create Date: 2016-06-28 15:44:52.141256
"""
# revision identifiers, used by Alembic.
revision = 'ab038e16ec9a'
down_revision = '4b8ce290f890'
branch_labels = None
depends_on = None
from alembic import op
import sqlalchemy as sa
def upgrade():
op.add_column('ofxaccount', sa.Column('deleted', sa.DateTime(), nullable=True))
op.add_column('ofxrecord', sa.Column('deleted', sa.DateTime(), nullable=True))
op.add_column('ofxsource', sa.Column('deleted', sa.DateTime(), nullable=True))
def downgrade():
op.drop_column('ofxsource', 'deleted')
op.drop_column('ofxrecord', 'deleted')
op.drop_column('ofxaccount', 'deleted')

11
templates/accounts.html
View File

@ -3,12 +3,19 @@
<h1>Accounts</h1>
{% if accounts %}
<table class="table">
<tr><th>Name</th><th>Type</th><th>Institution</th></tr>
<tr><th>Name</th><th>Type</th><th>Institution</th><th>Last Update</th><th></th></tr>
{% for account in accounts %}
<tr>
<td>{{ account.name }}</td>
<td>{{ account.type }}</td>
<td>{{ account.institution }}</td>
<td>{{ account.source.name }}</td>
<td>Never</td>
<td>
<form method="POST" action="/update/">
<input type="hidden" name="account_uuid" value="{{ account.uuid }}"></input>
<input type="submit" value="Update" class="btn btn-primary"></input>
</form>
</td>
</tr>
{% endfor %}
</table>

42
tests/files/query_transactions.ofx Normal file
View File

@ -0,0 +1,42 @@
OFXHEADER:100
DATA:OFXSGML
VERSION:102
SECURITY:NONE
ENCODING:USASCII
CHARSET:1252
COMPRESSION:NONE
OLDFILEUID:NONE
NEWFILEUID:NONE
<OFX>
<SIGNONMSGSRQV1>
<SONRQ>
<DTCLIENT>20160102030405.000[-7:MST]
<USERID>123456789
<USERPASS>1234
<LANGUAGE>ENG
<FI>
<ORG>AFCU
<FID>12345
</FI>
<APPID>QWIN
<APPVER>1200
</SONRQ>
</SIGNONMSGSRQV1>
<BANKMSGSRQV1>
<STMTTRNRQ>
<TRNUID>00000000
<STMTRQ>
<BANKACCTFROM>
<BANKID>1234567
<ACCTID>123456-0.9:CHK
<ACCTTYPE>CHECKING
</BANKACCTFROM>
<INCTRAN>
<DTSTART>20160102
<INCLUDE>Y
</INCTRAN>
</STMTRQ>
</STMTTRNRQ>
</BANKMSGSRQV1>
</OFX>

19
vanth/celery.py Normal file
View File

@ -0,0 +1,19 @@
import celery
import vanth.download
import vanth.main
import vanth.platform.ofxaccount
import vanth.platform.ofxrecord
import vanth.platform.ofxsource
app = celery.Celery('vanth')
app.conf.CELERY_ACCEPT_CONTENT = ['json', 'msgpack', 'yaml']
app.conf.CELERY_TASK_SERIALIZER = 'json'
app.conf.CELERY_ALWAYS_EAGER = True
@app.task()
def update_account(account_uuid):
account = vanth.platform.ofxaccount.by_uuid(account_uuid)[0]
source = vanth.platform.ofxsource.by_uuid(account['source']['uuid'])[0]
document = vanth.download.transactions(source, account)
vanth.platform.ofxrecord.ensure_exists(account, document.body.statement.transactions.items)

17
vanth/download.py Normal file
View File

@ -0,0 +1,17 @@
import requests
import vanth.ofx
import vanth.platform.ofxaccount
def do_all():
sources = {source['name']: source for source in vanth.platform.ofxsource.get()}
accounts = vanth.platform.ofxaccount.by_user(user_id=None)
for account in accounts:
transactions(sources[account['institution']], account)
def transactions(source, account):
body = vanth.ofx.query_transactions(source, account)
response = requests.post('https://ofx.americafirst.com/', data=body, headers={'Content-Type': 'application/x-ofx'})
assert response.ok, response.text
return vanth.ofx.parse(response.text)

39
vanth/pages/accounts.py
View File

@ -1,5 +1,7 @@
import flask
import vanth.celery
import vanth.pages.tools
import vanth.platform.ofxaccount
import vanth.platform.ofxsource
@ -7,27 +9,28 @@ blueprint = flask.Blueprint('accounts', __name__)
@blueprint.route('/accounts/', methods=['GET'])
def get_accounts():
my_accounts = vanth.platform.ofxaccount.get(flask.session['user_id'])
my_accounts = vanth.platform.ofxaccount.by_user(flask.session['user_id'])
sources = vanth.platform.ofxsource.get()
return flask.render_template('accounts.html', accounts=my_accounts, sources=sources)
@blueprint.route('/account/', methods=['POST'])
def post_account():
account_id = flask.request.form.get('account_id')
account_type = flask.request.form.get('account_type')
institution = flask.request.form.get('institution')
name = flask.request.form.get('name')
password = flask.request.form.get('password')
user_id = flask.request.form.get('user_id')
vanth.platform.ofxaccount.create({
'owner' : flask.session['user_id'],
'account_id' : account_id,
'institution' : institution,
'name' : name,
'password' : password,
'type' : account_type,
'user_id' : user_id,
@vanth.pages.tools.parse({
'account_id' : str,
'account_type' : str,
'institution' : str,
'name' : str,
'password' : str,
'user_id' : str,
})
def post_account(arguments):
arguments['owner'] = flask.session['user_id']
vanth.platform.ofxaccount.create(arguments)
return flask.redirect('/accounts/')
@blueprint.route('/update/', methods=['POST'])
@vanth.pages.tools.parse({
'account_uuid' : str,
})
def post_update(arguments):
vanth.celery.update_account.delay(**arguments)
return flask.redirect('/accounts/')

37
vanth/platform/ofxaccount.py
View File

@ -7,32 +7,49 @@ import vanth.platform.ofxsource
import vanth.tables
def get(user_id):
engine = chryso.connection.get()
query = sqlalchemy.select([
vanth.tables.OFXSource.c.name.label('institution'),
def _select():
return sqlalchemy.select([
vanth.tables.OFXAccount.c.account_id,
vanth.tables.OFXAccount.c.name,
vanth.tables.OFXAccount.c.password,
vanth.tables.OFXAccount.c.source,
vanth.tables.OFXAccount.c.type,
vanth.tables.OFXAccount.c.user_id,
vanth.tables.OFXAccount.c.uuid,
vanth.tables.OFXSource.c.name.label('source.name'),
vanth.tables.OFXSource.c.uuid.label('source.uuid'),
]).where(
vanth.tables.OFXAccount.c.source == vanth.tables.OFXSource.c.uuid
)
if user_id:
query = query.where(
vanth.tables.OFXAccount.c.owner == user_id
)
def _execute_and_convert(query):
engine = chryso.connection.get()
results = engine.execute(query)
return [{
'institution' : result[vanth.tables.OFXSource.c.name.label('institution')],
'account_id' : result[vanth.tables.OFXAccount.c.account_id],
'name' : result[vanth.tables.OFXAccount.c.name],
'source' : result[vanth.tables.OFXAccount.c.source],
'password' : result[vanth.tables.OFXAccount.c.password],
'source' : {
'name' : result[vanth.tables.OFXSource.c.name.label('source.name')],
'uuid' : result[vanth.tables.OFXSource.c.name.label('source.uuid')],
},
'type' : result[vanth.tables.OFXAccount.c.type],
'user_id' : result[vanth.tables.OFXAccount.c.user_id],
'uuid' : result[vanth.tables.OFXAccount.c.uuid],
} for result in results]
def by_uuid(account_uuid):
query = _select().where(vanth.tables.OFXAccount.c.uuid == account_uuid)
return _execute_and_convert(query)
def by_user(user_id):
query = _select()
if user_id:
query = query.where(
vanth.tables.OFXAccount.c.owner == user_id
)
return _execute_and_convert(query)
def create(values):
engine = chryso.connection.get()

34
vanth/platform/ofxrecord.py Normal file
View File

@ -0,0 +1,34 @@
import logging
import uuid
import chryso.connection
import sqlalchemy
import vanth.tables
LOGGER = logging.getLogger(__name__)
def ensure_exists(account, transactions):
engine = chryso.connection.get()
query = sqlalchemy.select([
vanth.tables.OFXRecord.c.fid,
]).where(vanth.tables.OFXRecord.c.ofxaccount == account['uuid'])
results = engine.execute(query).fetchall()
LOGGER.debug("Found %d OFX records for %s", len(results), account)
known_records = {result[vanth.tables.OFXRecord.c.fid] for result in results}
new_records = [transaction for transaction in transactions if transaction.id not in known_records]
LOGGER.debug("Have %d new transactions to save", len(new_records))
to_insert = [{
'amount' : transaction.amount,
'available' : transaction.available,
'fid' : transaction.id,
'memo' : transaction.memo,
'name' : transaction.name,
'ofxaccount' : account['uuid'],
'posted' : transaction.posted,
'type' : transaction.type,
'uuid' : uuid.uuid4(),
} for transaction in new_records]
if to_insert:
engine.execute(vanth.tables.OFXRecord.insert(), to_insert) # pylint: disable=no-value-for-parameter
LOGGER.debug("Done inserting %d records", len(new_records))

11
vanth/platform/ofxsource.py
View File

@ -6,8 +6,15 @@ import vanth.tables
LOGGER = logging.getLogger(__name__)
def get():
def _query_and_convert(query):
engine = chryso.connection.get()
query = vanth.tables.OFXSource.select()
results = engine.execute(query).fetchall()
return [dict(result) for result in results]
def by_uuid(uuid):
query = vanth.tables.OFXSource.select().where(vanth.tables.OFXSource.c.uuid == uuid)
return _query_and_convert(query)
def get():
query = vanth.tables.OFXSource.select()
return _query_and_convert(query)

37
vanth/tables.py
View File

@ -1,10 +1,22 @@
import chryso.constants
from sqlalchemy import (Column, Date, DateTime, Float, ForeignKey, Integer,
MetaData, String, Table, UniqueConstraint, func)
MetaData, String, Table, UniqueConstraint, func, text)
from sqlalchemy.dialects.postgresql import UUID
metadata = MetaData(naming_convention=chryso.constants.CONVENTION)
def table(name, *args, **kwargs):
return Table(
name,
metadata,
Column('uuid', UUID(as_uuid=True), primary_key=True, server_default=text('gen_random_uuid()')),
Column('created', DateTime, server_default=func.now(), nullable=False),
Column('updated', DateTime, server_default=func.now(), onupdate=func.now(), nullable=False),
Column('deleted', DateTime, nullable=True),
*args,
**kwargs
)
User = Table('users', metadata,
Column('uuid', UUID(), primary_key=True),
Column('username', String(255), nullable=False),
@ -17,8 +29,7 @@ User = Table('users', metadata,
UniqueConstraint('username', name='uq_user_username'),
)
CreditCard = Table('credit_card', metadata,
Column('uuid', UUID(as_uuid=True), primary_key=True),
CreditCard = table('credit_card',
Column('brand', String(20), nullable=False), # The brand of the card, like 'visa'
Column('card_id', String(100), nullable=False), # The ID of the card from Stripe
Column('country', String(1024), nullable=False), # The Country of the card, like 'US'
@ -28,23 +39,18 @@ CreditCard = Table('credit_card', metadata,
Column('last_four', Integer(), nullable=False), # The last four digits of the card
Column('token', String(), nullable=False), # The token we can use with Stripe to do stuff
Column('user_uri', String(2048), nullable=False), # The URI of the user that created the record
Column('created', DateTime(), nullable=False, server_default=func.now()),
Column('updated', DateTime(), nullable=False, server_default=func.now(), onupdate=func.now()),
Column('deleted', DateTime(), nullable=True),
UniqueConstraint('card_id', name='uq_credit_card_id'),
)
OFXSource = Table('ofxsource', metadata,
Column('uuid', UUID(as_uuid=True), primary_key=True),
OFXSource = table('ofxsource',
Column('name', String(255), nullable=False), # The name of the institution such as 'America First Credit Union'
Column('fid', String(255), nullable=False), # The FID of the institution, such as 54324
Column('bankid', String(255), nullable=False), # The bank ID of the institution such as 324377516. This may be a routing number
Column('created', DateTime(), nullable=False, server_default=func.now()),
Column('updated', DateTime(), nullable=False, server_default=func.now(), onupdate=func.now()),
Column('bankid', String(255), nullable=False), # The bank ID of the institution such as 324377516.
# This may be a routing number
UniqueConstraint('fid', name='uq_ofxsource_fid'),
)
OFXAccount = Table('ofxaccount', metadata,
OFXAccount = table('ofxaccount',
Column('account_id', String(255), nullable=False), # 123456-0.9:CHK
Column('name', String(255), nullable=False), # My checking account
Column('owner', None, ForeignKey(User.c.uuid, name='fk_user'), nullable=False),
@ -54,12 +60,9 @@ OFXAccount = Table('ofxaccount', metadata,
Column('user_id', String(255), nullable=False), # The user ID for the bank
Column('uuid', UUID(as_uuid=True), primary_key=True),
Column('created', DateTime(), nullable=False, server_default=func.now()),
Column('updated', DateTime(), nullable=False, server_default=func.now(), onupdate=func.now()),
)
OFXRecord = Table('ofxrecord', metadata,
Column('uuid', UUID(as_uuid=True), primary_key=True),
OFXRecord = table('ofxrecord',
Column('fid', String(255), nullable=False), # The Financial institution's ID
Column('amount', Float(), nullable=False), # The amount of the record, like -177.91
Column('available', Date(), nullable=True), # The date the record was available
@ -68,6 +71,4 @@ OFXRecord = Table('ofxrecord', metadata,
Column('memo', String(2048), nullable=True), # The memo of the transaction, like 'POINT OF SALE PURCHASE #0005727'
Column('type', String(255), nullable=True), # The type of the record, like 'POS'
Column('ofxaccount', None, ForeignKey(OFXAccount.c.uuid, name='fk_ofxaccount'), nullable=False),
Column('created', DateTime(), nullable=False, server_default=func.now()),
Column('updated', DateTime(), nullable=False, server_default=func.now(), onupdate=func.now()),
)