fetch: Loggerize, use argparse
parent
b8d4dc17e8
commit
05ac0356cf
|
|
@ -27,13 +27,15 @@
|
|||
# POSSIBILITY OF SUCH DAMAGE.
|
||||
|
||||
from datetime import date, datetime
|
||||
from getopt import getopt, GetoptError
|
||||
import sys
|
||||
from time import sleep, time
|
||||
import csv
|
||||
import os
|
||||
import random
|
||||
import re
|
||||
import logging
|
||||
import logging.config
|
||||
import argparse
|
||||
|
||||
import enum
|
||||
import bs4
|
||||
|
|
@ -52,6 +54,11 @@ config = {
|
|||
|
||||
Base = declarative_base()
|
||||
|
||||
if config.get('LOGGING'):
|
||||
logging.config.dictConfig(config['LOGGING'])
|
||||
else:
|
||||
logging.basicConfig(level=logging.DEBUG)
|
||||
|
||||
class RawTransfer(Base):
|
||||
__tablename__ = 'raw_transfer'
|
||||
id = Column(Integer, primary_key=True)
|
||||
|
|
@ -94,7 +101,7 @@ class IBRow(RawTransfer):
|
|||
return str(self)
|
||||
|
||||
def __init__(self, row, on_account, raw):
|
||||
self.raw = raw
|
||||
self.raw = raw.decode('utf-8')
|
||||
self.uid = row[IBField.uid]
|
||||
self.index = 1
|
||||
self.date = datetime.strptime(row[IBField.date_completed], "%Y%m%d").date()
|
||||
|
|
@ -254,6 +261,7 @@ class IBFetcher(object):
|
|||
BASE = "https://secure.ideabank.pl/"
|
||||
START_DATE = "01.11.2016"
|
||||
def __init__(self):
|
||||
self.logger = logging.getLogger(self.__class__.__name__)
|
||||
self._soup = None
|
||||
self.token = None
|
||||
self.s = requests.Session()
|
||||
|
|
@ -274,13 +282,14 @@ class IBFetcher(object):
|
|||
|
||||
def _dump(self):
|
||||
fn = config["DUMP_FILE"]
|
||||
print("[w] Dumping the last page to {}".format(fn))
|
||||
self.logger.warning("Dumping the last page to %f", fn)
|
||||
open(fn, 'w').write(unicode(self._soup).encode('utf-8'))
|
||||
|
||||
def _getraw(self, page, params={}):
|
||||
url = self.BASE + page
|
||||
r = self.s.get(url, params=params)
|
||||
print("[i] GET {}?{} -> {}".format(page, "&".join([str(k)+"="+str(v) for k,v in params.items()]), r.status_code))
|
||||
self.logger.debug("GET %s?%s -> %d", page, "&".join([
|
||||
str(k)+"="+str(v) for k, v in params.items()]), r.status_code)
|
||||
if r.status_code != 200:
|
||||
raise Exception("return code %i" % r.status_code)
|
||||
return r
|
||||
|
|
@ -302,7 +311,7 @@ class IBFetcher(object):
|
|||
|
||||
})
|
||||
r = self.s.post(url, data)
|
||||
print("[i] POST {} -> {}".format(page, r.status_code))
|
||||
self.logger.debug("POST %s -> %d", page, r.status_code)
|
||||
if r.status_code != 200:
|
||||
self._dump()
|
||||
raise Exception("return code %i" % r.status_code)
|
||||
|
|
@ -323,7 +332,7 @@ class IBFetcher(object):
|
|||
return soup
|
||||
|
||||
def _wait(self, seconds):
|
||||
print("[i] Waiting {} seconds".format(seconds))
|
||||
self.logger.debug("Waiting %d seconds", seconds)
|
||||
sleep(seconds)
|
||||
|
||||
def _gettoken(self, soup):
|
||||
|
|
@ -338,16 +347,16 @@ class IBFetcher(object):
|
|||
|
||||
if t is not None:
|
||||
self.token = t
|
||||
print("[i] Token: {}".format(self.token))
|
||||
self.logger.debug("Token: %s", self.token)
|
||||
else:
|
||||
print("[i] No new token found")
|
||||
self.logger.debug("No new token found")
|
||||
|
||||
def _hitjstoken(self, soup):
|
||||
m = re.search("\/main\/index\/token\/([0-9]+)\/time\/", str(soup.head))
|
||||
if m is not None:
|
||||
t = m.group(1)
|
||||
r = self._getraw("main/index/token/{}/time/{:.0f}.js".format(t, time()*1000), params={"t": "{:.16f}".format(random.random())})
|
||||
print("[i] Fetched JS timestamp token: \"{}\"".format(r.text))
|
||||
self.logger.debug("Fetched JS timestamp token: %r", r.text)
|
||||
|
||||
def process_wallet_page(self, soup):
|
||||
wallet = {"accounts": {}}
|
||||
|
|
@ -380,7 +389,7 @@ class IBFetcher(object):
|
|||
wallet["accounts"][account["number"]] = account
|
||||
|
||||
if len(wallet["accounts"]) == 0:
|
||||
print("[e] Empty accounts list. Undetected failed login? Aborting.")
|
||||
self.logger.error("Empty accounts list. Undetected failed login? Aborting.")
|
||||
self._dump()
|
||||
sys.exit(4)
|
||||
|
||||
|
|
@ -400,17 +409,10 @@ class IBFetcher(object):
|
|||
|
||||
password2_input = login2_page.find("input", attrs={"name": "password2"})
|
||||
if password2_input is None:
|
||||
print("[e] Masked password screen encountered - aborting")
|
||||
self.logger.error("Masked password screen encountered - aborting")
|
||||
sys.exit(4)
|
||||
#data["log"] = username
|
||||
#data["log2"] = ""
|
||||
#part_inputs = login2_page.find_all("input", class_="password_parts_inputs")
|
||||
#print("[i] Filling out {} characters".format(len(part_inputs)))
|
||||
#for input in part_inputs:
|
||||
#_,pos = input["name"].split("_")
|
||||
#data["pass_"+str(pos)] = password[int(pos)]
|
||||
else:
|
||||
print("[i] Regular password screen encountered")
|
||||
self.logger.debug("Regular password screen encountered")
|
||||
data["log2"] = username
|
||||
data["password2"] = password2_input["value"]
|
||||
data["password"] = password
|
||||
|
|
@ -418,10 +420,10 @@ class IBFetcher(object):
|
|||
wallet_page = self._post("main/index", data)
|
||||
|
||||
if wallet_page.find("div", class_="login_form"):
|
||||
print("[e] Login failed, aborting")
|
||||
self.logger.error("Login failed, aborting")
|
||||
self._dump()
|
||||
try:
|
||||
print("[e] Possible reason: {}".format(','.join(wallet_page.find("ul", class_="error_list").stripped_strings)))
|
||||
self.logger.error("Possible reason: %r", ','.join(wallet_page.find("ul", class_="error_list").stripped_strings))
|
||||
except:
|
||||
pass # screw it, we're fucked anyway
|
||||
sys.exit(4)
|
||||
|
|
@ -430,33 +432,6 @@ class IBFetcher(object):
|
|||
return self.process_wallet_page(wallet_page)
|
||||
|
||||
def fetch_account_history(self, account_id):
|
||||
#account_page = self._get("accounts/index/{}/2".format(account_id))
|
||||
#self._wait(4)
|
||||
|
||||
#data = {
|
||||
#"code": account_id,
|
||||
#"basic": 1,
|
||||
#"date_from": self.START_DATE,
|
||||
#"date_to": '{:02d}.{:02d}.{:04d}'.format(date.today().day, date.today().month, date.today().year),
|
||||
#"interval_time": "",
|
||||
#"interval_type": "",
|
||||
#"last": "",
|
||||
#"advanced[0]": "0",
|
||||
#"advanced[1]": "1",
|
||||
#"operation_type": "3",
|
||||
#"amount_from": "",
|
||||
#"amount_to": "",
|
||||
#"transaction_type": "",
|
||||
#"from": "",
|
||||
#"title": "",
|
||||
#"send": "send",
|
||||
#"ajaxSend": "true"
|
||||
#}
|
||||
#history_page = self._post("accounts/history/{}".format(account_id), data)
|
||||
#self._wait(2)
|
||||
|
||||
#r = self._getraw("accounts/printHistoryFile")
|
||||
|
||||
data = {
|
||||
"code": account_id,
|
||||
"report_type": "csv_dr",
|
||||
|
|
@ -475,143 +450,134 @@ def usage():
|
|||
def lock():
|
||||
fn = config["LOCK_FILE"]
|
||||
if os.path.isfile(fn):
|
||||
print("[e] Lock file {} exists, aborting".format(fn))
|
||||
logging.error("Lock file %s exists, aborting", fn)
|
||||
sys.exit(3)
|
||||
print("[i] Setting up lock file {}".format(fn))
|
||||
logging.debug("Setting up lock file %s", fn)
|
||||
open(fn,'w').close()
|
||||
if not os.path.isfile(fn):
|
||||
print("[e] Lock file {} somehow does not exist, aborting".format(fn))
|
||||
logging.error("Lock file %s somehow does not exist, aborting", fn)
|
||||
sys.exit(3)
|
||||
|
||||
def release():
|
||||
fn = config["LOCK_FILE"]
|
||||
print("[i] Removing lock file {}".format(fn))
|
||||
logging.debug("Removing lock file %s", fn)
|
||||
if not os.path.isfile(fn):
|
||||
print("[e] Lock file {} somehow does not exist, WTF?".format(fn))
|
||||
logging.error("Lock file %s somehow does not exist, WTF?", fn)
|
||||
sys.exit(3)
|
||||
os.remove(fn)
|
||||
if os.path.isfile(fn):
|
||||
print("[e] Lock file {} somehow still exists, WTF?".format(fn))
|
||||
logging.error("Lock file %s somehow still exists, WTF?", fn)
|
||||
sys.exit(3)
|
||||
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument('-n', '--no-action', action="store_true", help='do not commit any database changes')
|
||||
parser.add_argument('-c', '--cached', action="store_true", help='use cached data (test)')
|
||||
parser.add_argument('-l', '--load', action='append', help='process specified files (test)')
|
||||
parser.add_argument('--print-schema', action="store_true", help='print table schema and quit')
|
||||
|
||||
if __name__ == "__main__":
|
||||
try:
|
||||
opts, args = getopt(sys.argv[1:], "hcl:n", ["help", "cached", "load=", "no-action", "print-schema"])
|
||||
except GetoptError as err:
|
||||
# print help information and exit:
|
||||
print(str(err)) # will print something like "option -a not recognized"
|
||||
usage()
|
||||
sys.exit(2)
|
||||
args = parser.parse_args()
|
||||
|
||||
CACHE_DIR = config["CACHE_DIR"]
|
||||
engine = create_engine(config["SQLALCHEMY_DATABASE_URI"])
|
||||
session = sessionmaker(bind=engine)()
|
||||
|
||||
cached = False
|
||||
noaction = False
|
||||
load_files = {}
|
||||
for o, a in opts:
|
||||
if o in ("-h", "--help"):
|
||||
usage()
|
||||
sys.exit()
|
||||
elif o in ("--print-schema"):
|
||||
print("[i] Called with --print-schema, will print the create statement and quit.")
|
||||
m = MetaData()
|
||||
print(CreateTable(IBRow.__table__).compile(engine),";")
|
||||
for index in IBRow.__table__.indexes:
|
||||
print(CreateIndex(index).compile(engine),";")
|
||||
sys.exit()
|
||||
elif o in ("-c", "--cached"):
|
||||
cached = True
|
||||
elif o in ("-l", "--load"):
|
||||
an, f = a.split(":")
|
||||
if an is None or f is None:
|
||||
print("[e] --load argument \"{}\" appears malformed, could not split account number and file name".format(a))
|
||||
sys.exit(2)
|
||||
|
||||
account_number = IBParser.parse_account_number(an)
|
||||
if account_number is None:
|
||||
print("[e] Account number \"{}\" unparseable".format(an))
|
||||
sys.exit(2)
|
||||
|
||||
history = open(f,'r').read()
|
||||
load_files[account_number] = history
|
||||
cached = True
|
||||
print("[i] Loading \"{}\" as \"{}\"".format(f, account_number))
|
||||
elif o in ("-n", "--no-action"):
|
||||
print("[i] Called with --no-action, will not do any database operations.")
|
||||
noaction = True
|
||||
else:
|
||||
assert False, "unhandled option"
|
||||
if args.print_schema:
|
||||
logging.debug("Called with --print-schema, will print the create statement and quit.")
|
||||
m = MetaData()
|
||||
print('%s;' % CreateTable(IBRow.__table__).compile(engine))
|
||||
for index in IBRow.__table__.indexes:
|
||||
print('%s;' % CreateIndex(index).compile(engine))
|
||||
sys.exit()
|
||||
|
||||
lock()
|
||||
|
||||
balances = {}
|
||||
history_logs = {}
|
||||
|
||||
if cached:
|
||||
print("[i] Cached run - will not connect to the bank")
|
||||
if len(load_files) > 0:
|
||||
print("[i] Using manually supplied files")
|
||||
history_logs = load_files
|
||||
else:
|
||||
print("[i] Loading cached files from {}".format(CACHE_DIR))
|
||||
for f in os.listdir(CACHE_DIR):
|
||||
account_number = IBParser.parse_account_number(f)
|
||||
if account_number is None:
|
||||
print("[e] File name number \"{}\" unparseable".format(f))
|
||||
continue
|
||||
history = open(CACHE_DIR + "/" + f,'r').read()
|
||||
load_files[account_number] = history
|
||||
print("[i] Loading \"{}\" as \"{}\"".format(f, account_number))
|
||||
if len(load_files) == 0:
|
||||
print("[e] No cached files to process")
|
||||
sys.exit(2)
|
||||
history_logs = load_files
|
||||
if args.load:
|
||||
logging.debug("Using manually supplied files")
|
||||
for fn in args.load:
|
||||
an, f = fn.split(':')
|
||||
account_number = IBParser.parse_account_number(an)
|
||||
if account_number is None:
|
||||
logging.error("File name number \"{}\" unparseable".format(f))
|
||||
continue
|
||||
|
||||
logging.debug('Loading "%s" as "%s"', f, account_number)
|
||||
|
||||
with open(f, 'r') as fd:
|
||||
history_logs[account_number] = fd.read()
|
||||
|
||||
elif args.cached:
|
||||
logging.debug("Loading cached files from {}".format(CACHE_DIR))
|
||||
for f in os.listdir(CACHE_DIR):
|
||||
if f.startswith('balance-'):
|
||||
continue
|
||||
|
||||
account_number = IBParser.parse_account_number(f)
|
||||
if account_number is None:
|
||||
logging.error("File name number \"{}\" unparseable".format(f))
|
||||
continue
|
||||
|
||||
with open(CACHE_DIR + "/" + f,'r') as fd:
|
||||
history_logs[account_number] = fd.read()
|
||||
|
||||
logging.debug("Loading \"{}\" as \"{}\"".format(f, account_number))
|
||||
else:
|
||||
print("[i] Normal run - will connect to the bank")
|
||||
logging.debug("Normal run - will connect to the bank")
|
||||
fetcher = IBFetcher()
|
||||
history_logs = {}
|
||||
if "IB_LOGIN" not in config.keys() or "IB_PASSWORD" not in config.keys():
|
||||
wallet = fetcher.login(raw_input("[?] ID: "), raw_input("[?] Password: "))
|
||||
else:
|
||||
print("[i] Using saved credentials")
|
||||
logging.debug("Using saved credentials")
|
||||
wallet = fetcher.login(config["IB_LOGIN"], config["IB_PASSWORD"])
|
||||
|
||||
for account_number, account in wallet["accounts"].items():
|
||||
print("[i] Fetching history for account {} ({})".format(account_number, account["id"]))
|
||||
logging.debug("Fetching history for account {} ({})".format(account_number, account["id"]))
|
||||
history = fetcher.fetch_account_history(account["id"])
|
||||
cachefile = open(CACHE_DIR+"/"+account_number,'w')
|
||||
cachefile.write(history)
|
||||
cachefile.close()
|
||||
|
||||
history_logs[account_number] = history
|
||||
balances[account_number] = (account["available_balance"],account["currency"])
|
||||
balancefile = open(CACHE_DIR+"/balance-"+account_number,'w')
|
||||
balancefile.write("{} {}\n".format(account["available_balance"],account["currency"]))
|
||||
balancefile.close()
|
||||
with open(CACHE_DIR+"/"+account_number,'w') as fd:
|
||||
fd.write(history)
|
||||
|
||||
balances[account_number] = (account["available_balance"], account["currency"])
|
||||
with open(CACHE_DIR+"/balance-"+account_number,'w') as fd:
|
||||
fd.write("{} {}\n".format(
|
||||
account["available_balance"],account["currency"]))
|
||||
|
||||
if not history_logs:
|
||||
logging.error('Nothing to process')
|
||||
sys.exit()
|
||||
|
||||
parsed = {}
|
||||
stats = {}
|
||||
for account_number, history in history_logs.items():
|
||||
print("[i] Parsing history for account {}".format(account_number))
|
||||
logging.debug("Parsing history for account {}".format(account_number))
|
||||
parser = IBParser(account_number)
|
||||
parser.parse(history)
|
||||
stats[account_number] = {}
|
||||
stats[account_number]["added"] = 0
|
||||
stats[account_number]["skipped"] = 0
|
||||
for row in parser.get():
|
||||
if not session.query(IBRow).filter_by(uid=row.uid).first() and not noaction:
|
||||
if not session.query(IBRow).filter_by(uid=row.uid).first():
|
||||
session.add(row)
|
||||
stats[account_number]["added"] += 1
|
||||
else:
|
||||
stats[account_number]["skipped"] += 1
|
||||
if noaction:
|
||||
print("[i] --no-action set, skipping row {}".format(row))
|
||||
session.commit()
|
||||
|
||||
if args.no_action:
|
||||
logging.info('Running with --no-action, not commiting.')
|
||||
else:
|
||||
session.commit()
|
||||
|
||||
if balances:
|
||||
print("[i] Account balances:")
|
||||
logging.debug("Account balances:")
|
||||
for account_number,v in balances.items():
|
||||
balance,currency = v
|
||||
print("\t{}: {} {}".format(account_number, balance, currency))
|
||||
print("[i] Done: ", stats)
|
||||
logging.debug("\t{}: {} {}".format(account_number, balance, currency))
|
||||
|
||||
if any(v['added'] for v in stats.values()):
|
||||
logging.info("Done: %r", stats)
|
||||
|
||||
release()
|
||||
|
|
|
|||
|
|
@ -2,6 +2,44 @@ class Config(object):
|
|||
DEBUG = False
|
||||
TESTING = False
|
||||
SQLALCHEMY_DATABASE_URI = "sqlite:///data.db"
|
||||
OWN_ACCOUNTS = []
|
||||
SECRET = 'setme'
|
||||
|
||||
CACHE_DIR = 'cache/'
|
||||
LOCK_FILE = '/tmp/kasownik.lock'
|
||||
|
||||
LOGGING = {
|
||||
'version': 1,
|
||||
'disable_existing_loggers': False,
|
||||
'formatters': {
|
||||
'standard': {
|
||||
'format': '%(asctime)s [%(levelname)s] %(name)s: %(message)s',
|
||||
'datefmt': '%Y-%m-%d %H:%M:%S',
|
||||
},
|
||||
},
|
||||
'handlers': {
|
||||
'console': {
|
||||
'class': 'logging.StreamHandler',
|
||||
'level': 'INFO',
|
||||
'formatter': 'standard',
|
||||
},
|
||||
'file': {
|
||||
'class': 'logging.handlers.TimedRotatingFileHandler',
|
||||
'level': 'DEBUG',
|
||||
'formatter': 'standard',
|
||||
'filename': 'kasownik.log',
|
||||
'backupCount': 7,
|
||||
'when': 'midnight',
|
||||
}
|
||||
},
|
||||
'loggers': {
|
||||
'': {
|
||||
'handlers': ['console', 'file'],
|
||||
'level': 'DEBUG',
|
||||
'propagate': True,
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
class DevelopmentConfig(Config):
|
||||
|
|
|
|||
Loading…
Reference in New Issue