From 393ef748ccb338c65498c456efd3e9311ca5dcdd Mon Sep 17 00:00:00 2001 From: Eli Ribble Date: Thu, 11 Aug 2016 10:54:29 -0600 Subject: [PATCH] Get OFX file upload working through celery I switched to using ofxparse because my own parser wasn't up to snuff on handling the download file that I got from my bank and I got sick of maintaining my own because SGML is an ugly mess This commit means that my automatic download won't work any more because it's expecting to pass data through my own parser, which we're not going to do any more. That's okay because my bank's OFX integration is actually down anyways --- setup.py | 1 + vanth/celery.py | 22 ++++++++++++++++++---- vanth/pages/accounts.py | 1 + vanth/platform/ofxrecord.py | 5 ++--- 4 files changed, 22 insertions(+), 7 deletions(-) diff --git a/setup.py b/setup.py index 569559f..72ffc48 100644 --- a/setup.py +++ b/setup.py @@ -101,6 +101,7 @@ def main(): 'chryso==1.7', 'Flask==0.10.1', 'flask-login==0.3.2', + 'ofxparse==0.15', 'sepiida==5.27', ], extras_require = { diff --git a/vanth/celery.py b/vanth/celery.py index 2c0dec9..8e12e24 100644 --- a/vanth/celery.py +++ b/vanth/celery.py @@ -1,9 +1,12 @@ import logging import celery +import ofxparse +import requests.exceptions import vanth.download import vanth.main +import vanth.ofx import vanth.platform.ofxaccount import vanth.platform.ofxrecord import vanth.platform.ofxsource @@ -14,13 +17,24 @@ LOGGER = logging.getLogger(__name__) app = celery.Celery('vanth') app.conf.CELERY_ACCEPT_CONTENT = ['json', 'msgpack', 'yaml'] app.conf.CELERY_TASK_SERIALIZER = 'json' -app.conf.CELERY_ALWAYS_EAGER = True +#app.conf.CELERY_ALWAYS_EAGER = True @app.task() def update_account(account_uuid): LOGGER.debug("Updating account %s", account_uuid) account = vanth.platform.ofxaccount.by_uuid(account_uuid) source = vanth.platform.ofxsource.by_uuid(account['source']['uuid'])[0] - document = vanth.download.transactions(source, account) - vanth.platform.ofxrecord.ensure_exists(account, document.body.statement.transactions.items) - vanth.platform.ofxupdate.OFXUpdate.create(ofxaccount=account_uuid) + try: + document = vanth.download.transactions(source, account) + vanth.platform.ofxrecord.ensure_exists(account, document.body.statement.transactions.items) + vanth.platform.ofxupdate.OFXUpdate.create(ofxaccount=account_uuid) + except requests.exceptions.ConnectionError as e: + LOGGER.error("Failed to download account data: %s", e) + +@app.task() +def process_transaction_upload(account_uuid, filename): + LOGGER.debug("Processing file %s", filename) + account = vanth.platform.ofxaccount.by_uuid(account_uuid) + with open(filename, 'rb') as f: + document = ofxparse.OfxParser.parse(f) + vanth.platform.ofxrecord.ensure_exists(account, document.account.statement.transactions) diff --git a/vanth/pages/accounts.py b/vanth/pages/accounts.py index fc45523..0908671 100644 --- a/vanth/pages/accounts.py +++ b/vanth/pages/accounts.py @@ -58,4 +58,5 @@ def post_transactions(arguments): filename = werkzeug.utils.secure_filename(transactions.filename) LOGGER.info("Saving uploaded file %s to %s", transactions.filename, filename) transactions.save(filename) + vanth.celery.process_transaction_upload.delay(arguments['account_uuid'], filename) return flask.redirect('/accounts/{}/'.format(arguments['account_uuid'])) diff --git a/vanth/platform/ofxrecord.py b/vanth/platform/ofxrecord.py index afff768..3c517ea 100644 --- a/vanth/platform/ofxrecord.py +++ b/vanth/platform/ofxrecord.py @@ -20,12 +20,11 @@ def ensure_exists(account, transactions): LOGGER.debug("Have %d new transactions to save", len(new_records)) to_insert = [{ 'amount' : transaction.amount, - 'available' : transaction.available, 'fid' : transaction.id, 'memo' : transaction.memo, - 'name' : transaction.name, + 'name' : transaction.payee, 'ofxaccount' : account['uuid'], - 'posted' : transaction.posted, + 'posted' : transaction.date, 'type' : transaction.type, 'uuid' : uuid.uuid4(), } for transaction in new_records]