Removed BRE/mBank support. Good riddance. ~enleth
Kasownik
parent
a49283cbe3
commit
3f7ef801f9
|
|
@ -1,238 +0,0 @@
|
|||
#!/usr/bin/env/python2
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
# Copyright (c) 2015, Sergiusz Bazanski <q3k@q3k.org>
|
||||
# All rights reserved.
|
||||
#
|
||||
# Redistribution and use in source and binary forms, with or without
|
||||
# modification, are permitted provided that the following conditions are met:
|
||||
#
|
||||
# 1. Redistributions of source code must retain the above copyright notice,
|
||||
# this list of conditions and the following disclaimer.
|
||||
# 2. Redistributions in binary form must reproduce the above copyright notice,
|
||||
# this list of conditions and the following disclaimer in the documentation
|
||||
# and/or other materials provided with the distribution.
|
||||
#
|
||||
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
||||
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
|
||||
# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
|
||||
# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
|
||||
# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
|
||||
# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
|
||||
# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
|
||||
# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
|
||||
# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
|
||||
# POSSIBILITY OF SUCH DAMAGE.
|
||||
|
||||
import csv
|
||||
import datetime
|
||||
import re
|
||||
import hashlib
|
||||
import StringIO
|
||||
import requests
|
||||
import bs4
|
||||
import time
|
||||
|
||||
from webapp import app
|
||||
|
||||
|
||||
class BREParseError(Exception):
|
||||
pass
|
||||
|
||||
|
||||
class BRERow(object):
|
||||
BRE_IN = [772, 773, 770, 968, 958, 963, 592]
|
||||
SECRET = app.config["SECRET"]
|
||||
|
||||
def parse_data(self):
|
||||
datar = self.data_raw.split(";")
|
||||
data = {}
|
||||
for d in datar[1:]:
|
||||
kv = d.split(":")
|
||||
k = kv[0].strip()
|
||||
v = ":".join(kv[1:]).strip()
|
||||
data[k] = v
|
||||
|
||||
if self._type in self.BRE_IN:
|
||||
# in
|
||||
try:
|
||||
# internal transfers use the same type ID for both in and out
|
||||
if "dla" in data:
|
||||
return
|
||||
self.type = "IN"
|
||||
|
||||
self.from_name = data["od"]
|
||||
self.from_account = data["z rach."]
|
||||
if "tyt." in data:
|
||||
self.title = data["tyt."].lower()
|
||||
elif "ref. zlec." in data:
|
||||
self.title = data["ref. zlec."].lower()
|
||||
else:
|
||||
self.title = "[no transfer title]"
|
||||
self.tnr = int(data["TNR"].split(".")[0])
|
||||
except KeyError:
|
||||
print "Error parsing transfer data ", data
|
||||
raise
|
||||
|
||||
self.olduid = hashlib.sha256(self.SECRET + ','.join(self.raw).encode("utf-8")).hexdigest()
|
||||
self.uid = hashlib.sha256(self.SECRET + data["TNR"]).hexdigest()
|
||||
|
||||
def __init__(self, row):
|
||||
self.time = datetime.datetime.strptime(row[1], "%d/%m/%Y")
|
||||
self.account = row[2]
|
||||
# is this secure?
|
||||
self.amount = int(float(row[3].replace(",", ".").replace(" ", "")) * 100)
|
||||
self._type = int(row[6])
|
||||
self.data_raw = row[5]
|
||||
|
||||
self.type = ""
|
||||
self.raw = row
|
||||
|
||||
|
||||
class BREParser(object):
|
||||
def __init__(self):
|
||||
self.rows = []
|
||||
|
||||
def parse(self, snapshot):
|
||||
c = csv.reader(snapshot, delimiter="|")
|
||||
for row in c:
|
||||
r = BRERow([r.decode("iso-8859-2") for r in row])
|
||||
r.parse_data()
|
||||
self.rows.append(r)
|
||||
|
||||
def get_by_type(self, y):
|
||||
return [row for row in self.rows if row.type == y]
|
||||
|
||||
|
||||
def guess_title(title):
|
||||
m = re.match(ur"^([a-z0-9\-_\.]+) *\- *(fatty|starving) *z\- *([0-9a-z\-_ąężźćóżłśń]+$)", title.strip().lower())
|
||||
if not m:
|
||||
return None, None, None
|
||||
member, _type, title = m.group(1), m.group(2), m.group(3)
|
||||
if title in [u"składka", u"opłata", u"opłata miesięczna", "skladka"]:
|
||||
return member, _type, None
|
||||
return member, _type, title
|
||||
|
||||
class BREFetcher(object):
|
||||
BASE = "https://companynet.mbank.pl/mt/"
|
||||
def __init__(self):
|
||||
self.uid = app.config["BRE_UID"]
|
||||
self.logging_token = None
|
||||
self.token = None
|
||||
self.s = requests.Session()
|
||||
|
||||
def _get(self, page):
|
||||
url = self.BASE + page
|
||||
#r = self.s.get(url,verify=app.config["BRE_CA_PATH"])
|
||||
r = self.s.get(url)
|
||||
print "[i] GET {} -> {}".format(page, r.status_code)
|
||||
if r.status_code != 200:
|
||||
raise Exception("return code %i" % r.status_code)
|
||||
return bs4.BeautifulSoup(r.text)
|
||||
|
||||
def _post(self, page, data):
|
||||
url = self.BASE + page
|
||||
mdata = {}
|
||||
mdata["screenWidth"] = 1337
|
||||
mdata["screenHeight"] = 1337
|
||||
mdata["LOGGING_TOKEN"] = self.logging_token
|
||||
mdata["lang"] = ""
|
||||
mdata.update(data)
|
||||
r = self.s.post(url, mdata)
|
||||
print "[i] POST {} -> {}".format(page, r.status_code)
|
||||
if r.status_code != 200:
|
||||
raise Exception("return code %i" % r.status_code)
|
||||
return bs4.BeautifulSoup(r.text)
|
||||
|
||||
def _gettoken(self, soup):
|
||||
# print soup
|
||||
menulinks = soup.findAll("a", "menulink")
|
||||
onclick = menulinks[0]["onclick"]
|
||||
self.token = re.search(r"TOKEN=([a-z0-9]+)", onclick).group(1)
|
||||
print "[i] Token: {}".format(self.token)
|
||||
|
||||
def login(self, username, token):
|
||||
main = self._get("fragments/cua/login.jsp")
|
||||
self.logging_token = main.find("input", type="hidden", attrs={"name": "LOGGING_TOKEN"})["value"]
|
||||
|
||||
data = {}
|
||||
data["TARGET"] = "/cualogin.do"
|
||||
data["RAWPASSWORD"] = token
|
||||
data["loginType"] = "token"
|
||||
data["LOGIN_OR_ALIAS"] = username
|
||||
logged = self._post("fragments/cua/login.fcc", data)
|
||||
self._gettoken(logged)
|
||||
|
||||
def create_report(self):
|
||||
reportpage = self._get("main/navigate.do?templateId={}&to=newReport&org.apache.struts.taglib.html.TOKEN={}".format(self.uid, self.token))
|
||||
self._gettoken(reportpage)
|
||||
|
||||
def generate_command(command, commandlist):
|
||||
data = {
|
||||
"synchronous": "on",
|
||||
"pagerOffset": 0,
|
||||
"pager.page": 0,
|
||||
"pager.newPage": 0,
|
||||
"org.apache.struts.taglib.html.TOKEN": self.token,
|
||||
"filter.orderByColumn": None,
|
||||
"filter.orderByAscending": "false",
|
||||
"commandlist": commandlist,
|
||||
"command": command,
|
||||
}
|
||||
return data
|
||||
|
||||
def setparameter(item, value, extra=None):
|
||||
data = generate_command("editItem", "generate")
|
||||
data["selectedItemKey"] = item
|
||||
|
||||
generate = self._post("report/generate/submitParameterList.do", data)
|
||||
self._gettoken(generate)
|
||||
|
||||
data = {}
|
||||
data["value"] = value
|
||||
data["org.apache.struts.taglib.html.TOKEN"] = self.token
|
||||
data["commandlist"] = "ok"
|
||||
data["command"] = "ok"
|
||||
if extra:
|
||||
data.update(extra)
|
||||
|
||||
specify = self._post("report/specify/submitParameter.do", data)
|
||||
self._gettoken(specify)
|
||||
|
||||
setparameter("{}.0[account]".format(self.uid), 1060690633)
|
||||
setparameter("{}.0[from_date]".format(self.uid), "01.01.2001 00:00", {"predefiniedValue.selectedKey": "@empty", "valueTimePart": "00:00", "valueDatePart": "01.01.2001"})
|
||||
setparameter("{}.0[to_date]".format(self.uid), "", {"predefiniedValue.selectedKey": "@currentday", "valueTimePart": "", "valueDatePart": ""})
|
||||
|
||||
data = generate_command("generate", "generate")
|
||||
data["selectedItemKey"] = "{}.0[to_date]".format(self.uid)
|
||||
submit_parameter_list = self._post("report/generate/submitParameterList.do", data)
|
||||
self._gettoken(submit_parameter_list)
|
||||
print "[i] Waiting..."
|
||||
time.sleep(3)
|
||||
data = {}
|
||||
data["org.apache.struts.taglib.html.TOKEN"] = self.token
|
||||
data["commandlist"] = "showReport"
|
||||
data["command"] = "showReport"
|
||||
report = self._post("report/generate/submitPleaseWait.do", data)
|
||||
self._gettoken(report)
|
||||
|
||||
data = generate_command("export", "export")
|
||||
report_ready = self._post("report/submitReportReady.do", data)
|
||||
self._gettoken(report_ready)
|
||||
|
||||
data = generate_command("processExport", "processExport")
|
||||
data["format.selectedKey"] = "dat"
|
||||
data["fileName"] = "raport"
|
||||
data["encoding.selectedKey"] = "iso"
|
||||
export = self._post("report/submitExport.do", data)
|
||||
self._gettoken(export)
|
||||
fileurl = "report/downloadExport.do?command=download&FileNumber=0"
|
||||
f = self.s.post(self.BASE + fileurl, dict(FileNumber=0), stream=True)
|
||||
return f.raw
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
f = BREFetcher()
|
||||
f.login(raw_input("[?] ID: "), raw_input("[?] Token: "))
|
||||
print f.create_report().read()
|
||||
|
|
@ -37,10 +37,6 @@ class ContactEmailSettingsForm(Form):
|
|||
ldap = BooleanField("")
|
||||
custom = TextField("Custom address:")
|
||||
|
||||
class BREFetchForm(Form):
|
||||
identifier = TextField("Identifier", [validators.Required()])
|
||||
token = PasswordField("Identifier", [validators.Required()])
|
||||
|
||||
class LDAPSyncForm(Form):
|
||||
fatty_to_add = MultiCheckboxField("Fatty to add", choices=[])
|
||||
fatty_to_remove = MultiCheckboxField("Fatty to remove", choices=[])
|
||||
|
|
|
|||
|
|
@ -22,33 +22,10 @@
|
|||
# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
|
||||
# POSSIBILITY OF SUCH DAMAGE.
|
||||
|
||||
"""The 'business' logic of the whole thing.
|
||||
It updates database transfer rows based on BRE snapshots, and it also
|
||||
tries to match transfers onto members, updating their payment status."""
|
||||
"""The 'business' logic of the whole thing."""
|
||||
|
||||
from webapp import app, db
|
||||
import banking
|
||||
import models
|
||||
|
||||
|
||||
def update_transfer_rows():
|
||||
f = open(app.config["BRE_SNAPSHOT_PATH"])
|
||||
parser = banking.BREParser()
|
||||
parser.parse(f)
|
||||
f.close()
|
||||
|
||||
for row in parser.get_by_type("IN"):
|
||||
transfer = models.Transfer.query.filter_by(uid=row.uid).first()
|
||||
if not transfer:
|
||||
transfer = models.Transfer(None, row.uid,
|
||||
row.from_account,
|
||||
row.from_name,
|
||||
row.amount, row.title,
|
||||
row.time,
|
||||
False)
|
||||
db.session.add(transfer)
|
||||
db.session.commit()
|
||||
|
||||
|
||||
def get_unmatched_transfers():
|
||||
return models.Transfer.query.filter_by(member_transfers=None,ignore=False).order_by(models.Transfer.date.asc()).all()
|
||||
|
|
|
|||
|
|
@ -9,13 +9,7 @@
|
|||
<h4>Active operations:</h4>
|
||||
<h4>Available operations:</h4>
|
||||
<p>
|
||||
<!--<form action="/fetch" method="post">
|
||||
<button type="button" class="btn btn-primary">Fetch transfer data</button>
|
||||
</form>
|
||||
<form action="/spam" method="post">
|
||||
<button type="button" class="btn btn-primary">Send reminders</button>
|
||||
</form>-->
|
||||
<a href="/admin/fetch"><b>Fetch transfer data</b></a>
|
||||
<a href="/admin/match"><b>Match transfers</b></a>
|
||||
<a href="/admin/ldapsync"><b>Synchronize LDAP groups</b></a>
|
||||
<a href="/admin/spam"><b>Spam members</b></a>
|
||||
</p>
|
||||
|
|
|
|||
|
|
@ -9,13 +9,7 @@
|
|||
<h4>Active operations:</h4>
|
||||
<h4>Available operations:</h4>
|
||||
<p>
|
||||
<!--<form action="/fetch" method="post">
|
||||
<button type="button" class="btn btn-primary">Fetch transfer data</button>
|
||||
</form>
|
||||
<form action="/spam" method="post">
|
||||
<button type="button" class="btn btn-primary">Send reminders</button>
|
||||
</form>-->
|
||||
<a href="/admin/fetch"><b>Fetch transfer data</b></a>
|
||||
<a href="/admin/match"><b>Match transfers</b></a>
|
||||
<a href="/admin/ldapsync"><b>Synchronize LDAP groups</b></a>
|
||||
</p>
|
||||
</div>
|
||||
|
|
|
|||
|
|
@ -1,14 +1,9 @@
|
|||
{% extends "root.html" %}
|
||||
{% block title %}brefetch{% endblock %}
|
||||
{% block title %}Match transfers{% endblock %}
|
||||
{% block content %}
|
||||
|
||||
<div class="container">
|
||||
<h2>Fetch banking data from BRE</h2>
|
||||
<form method="post" action="/admin/fetch">
|
||||
{{ form.identifier()|safe }}
|
||||
{{ form.token()|safe }}
|
||||
<input type="submit" value="Fetch" />
|
||||
</form>
|
||||
<h2>Match transfers</h2>
|
||||
<h2>Matching operations</h2>
|
||||
<a href="/admin/match/auto">Match all easily matchable transfers</a><br />
|
||||
<a href="/admin/match/manual">Match manually all unmatched transfers</a><br />
|
||||
|
|
@ -35,7 +35,6 @@ from subprocess import Popen, PIPE
|
|||
from webapp import app, forms, User, db, models, mc, cache_enabled, admin_required
|
||||
from flask.ext.login import login_user, login_required, logout_user, current_user
|
||||
from flask import Response, request, redirect, flash, render_template, url_for, abort, g
|
||||
import banking
|
||||
import logic
|
||||
import directory
|
||||
import traceback
|
||||
|
|
@ -214,33 +213,12 @@ def add_member(membershiptype, username):
|
|||
db.session.commit()
|
||||
return "ok"
|
||||
|
||||
@app.route("/admin/fetch", methods=["GET", "POST"])
|
||||
@app.route("/admin/match")
|
||||
@login_required
|
||||
@admin_required
|
||||
def admin_fetch():
|
||||
form = forms.BREFetchForm(request.form)
|
||||
if request.method == "POST" and form.validate():
|
||||
identifier = form.identifier.data
|
||||
token = form.token.data
|
||||
try:
|
||||
b = banking.BREFetcher()
|
||||
b.login(identifier, token)
|
||||
data = b.create_report().read()
|
||||
flash("Fetched data from BRE ({} rows)".format(data.count("\n")))
|
||||
f = open(app.config["BRE_SNAPSHOT_PATH"], "w")
|
||||
f.write(data)
|
||||
f.close()
|
||||
return redirect(url_for("admin_fetch"))
|
||||
except Exception as e:
|
||||
exc_type, exc_value, exc_traceback = sys.exc_info()
|
||||
|
||||
flash("Error when fetching data. <pre>%s</pre>" % traceback.format_exception(exc_type, exc_value,exc_traceback))
|
||||
return redirect(url_for("admin_fetch"))
|
||||
|
||||
logic.update_transfer_rows()
|
||||
def admin_match():
|
||||
transfers_unmatched = logic.get_unmatched_transfers()
|
||||
|
||||
return render_template("fetch.html", form=form, transfers_unmatched=transfers_unmatched)
|
||||
return render_template("match.html", transfers_unmatched=transfers_unmatched)
|
||||
|
||||
|
||||
@app.route("/admin/match/auto", methods=["GET"])
|
||||
|
|
@ -280,7 +258,7 @@ def admin_match_auto():
|
|||
for member in affected_members:
|
||||
member.get_status(force_refresh=True)
|
||||
flash("Matched %i, %i left" % (matched, left))
|
||||
return redirect(url_for("admin_fetch"))
|
||||
return redirect(url_for("admin_match"))
|
||||
|
||||
@app.route("/admin/match/manual", methods=["GET"])
|
||||
@login_required
|
||||
|
|
@ -349,3 +327,11 @@ def logout():
|
|||
return redirect(url_for("stats"))
|
||||
|
||||
|
||||
from prometheus_client import multiprocess
|
||||
from prometheus_client import generate_latest, CollectorRegistry, CONTENT_TYPE_LATEST
|
||||
@app.route("/varz")
|
||||
def metrics():
|
||||
registry = CollectorRegistry()
|
||||
multiprocess.MultiProcessCollector(registry)
|
||||
data = generate_latest(registry)
|
||||
return Response(data, mimetype=CONTENT_TYPE_LATEST)
|
||||
|
|
|
|||
Loading…
Reference in New Issue