Merge branch 'docker-deployment'

.gitignore

@@ -1,6 +1,6 @@
 olddata
 webapp/data.db
-config.py
+web/webapp/data.db
 *pyc
 *sublime*
 kasownik.ini

Dockerfile

@@ -0,0 +1,17 @@
+FROM python:3.9-slim
+
+RUN apt-get update && apt-get install --no-install-recommends -y build-essential libsasl2-dev libldap2-dev
+
+ADD web/requirements.txt .
+
+# Downgrade setuptools to fix anyjson dependency
+RUN pip install setuptools~=57.5.0 && \
+    pip install -r requirements.txt
+
+WORKDIR /usr/src/web
+
+ADD web /usr/src/web
+ADD fetch /usr/src/fetch
+
+STOPSIGNAL SIGINT
+CMD ["uwsgi", "--http-socket", "0.0.0.0:5000", "--plugins", "python3", "--wsgi", "webapp.wsgi:app", "--threads", "10", "--master"]

README.md

@@ -14,3 +14,12 @@ This project is divided into two separate modules:
     (at least separate UID) - supports "old" IdeaBank web interface
 
 More info about these can be found in their respective `README.md` files.
+
+Quick Start
+-----------
+
+1. [Register new SSO application](https://sso.hackerspace.pl/client/create) - client name and URI don't matter, redirect URI should be `http://localhost:5000/oauth/callback` (by default), other settings can stay default
+2. Set `SPACEAUTH_CONSUMER_KEY` and `SPACEAUTH_CONSUMER_SECRET` envs to the client generated above
+3. `docker-compose run --rm kasownik-web ./manage.py syncdb` (one time)
+4. Run the app: `docker-compose up --build`
+5. (TODO: Add missing table for fetcher, add example data)

docker-compose.yml

@@ -0,0 +1,14 @@
+version: "3.8"
+
+services:
+  kasownik-web:
+    build: .
+    ports:
+      - 5000:5000
+    volumes:
+      - ./web:/usr/src/web
+      - ./fetch:/usr/src/fetch
+    environment:
+      - SPACEAUTH_CONSUMER_KEY
+      - SPACEAUTH_CONSUMER_SECRET
+      - DISABLE_LDAP=true

fetch/banking-pekaobiznes.py

@@ -371,7 +371,7 @@ def release(fn):
         sys.exit(3)
 
 parser = argparse.ArgumentParser()
-parser.add_argument('--config', help="Load configuration file", default="config.ini")
+parser.add_argument('--config', help="Load configuration file")
 parser.add_argument('-n', '--no-action', action="store_true", help='do not commit any database changes')
 parser.add_argument('-c', '--cached', action="store_true", help='use cached data (test)')
 parser.add_argument('-l', '--load', action='append', help='process specified files (test)')
@@ -382,8 +382,19 @@ parser.add_argument('--print-schema', action="store_true", help='print table sch
 if __name__ == "__main__":
     args = parser.parse_args()
 
-    config = configparser.ConfigParser()
-    config.read(args.config)
+    config = configparser.ConfigParser(defaults=os.environ, interpolation=configparser.ExtendedInterpolation())
+    config.read_dict({
+        'logging': {
+            'level': 'INFO',
+        },
+        'general': {
+            'cache_dir': 'cache',
+            'lockfile': 'lockfile',
+        },
+    })
+
+    if args.config:
+        config.read(args.config)
 
     logging.basicConfig(level=config['logging']['level'], format=config['logging'].get('format', '%(asctime)s [%(levelname)s] %(name)s: %(message)s'))
     logging.getLogger('chardet').setLevel(logging.WARN)

web/config.py

@@ -0,0 +1,36 @@
+import environs
+
+env = environs.Env()
+env.read_env()
+
+DEBUG = env.bool("DEBUG", False)
+SQLALCHEMY_DATABASE_URI = env.str("SQLALCHEMY_DATABASE_URI", "sqlite:///data.db")
+DUMMY_TRANSFER_UID = "NOTAMEMBER"
+
+SPACEAUTH_CONSUMER_KEY = env.str("SPACEAUTH_CONSUMER_KEY", "kasownik")
+SPACEAUTH_CONSUMER_SECRET = env.str("SPACEAUTH_CONSUMER_SECRET", "changeme")
+
+SECRET_KEY = env.str("SECRET_KEY", "changeme")
+
+DISABLE_LDAP = env.bool("DISABLE_LDAP", False)
+
+LDAP_URI = env.str("LDAP_URI", "ldaps://ldap.hackerspace.pl")
+LDAP_BIND_DN = env.str("LDAP_BIND_DN", "cn=kasownik,ou=Services,dc=hackerspace,dc=pl")
+LDAP_BIND_PASSWORD = env.str("LDAP_BIND_PASSWORD", "changeme")
+LDAP_CA_PATH = env.str("LDAP_CA_PATH", "/etc/ssl/certs/ca-certificates.crt")
+
+LDAP_USER_FILTER = env.str("LDAP_USER_FILTER", "(objectClass=hsMember)")
+LDAP_USER_BASE = env.str("LDAP_USER_BASE", "ou=People,dc=hackerspace,dc=pl")
+LDAP_GROUP_FILTER = env.str("LDAP_GROUP_FILTER", "(objectClass=groupOfUniqueNames)")
+LDAP_GROUP_BASE = env.str("LDAP_GROUP_BASE", "ou=Group,dc=hackerspace,dc=pl")
+
+CACHE_TYPE = env.str("CACHE_TYPE", "null")
+CACHE_NO_NULL_WARNING = True
+
+MEMBERSHIP_FEES = env.json(
+    "MEMBERSHIP_FEES",
+    {
+        "starving": 75,
+        "fatty": 150,
+    },
+)

web/config.py.dist

@@ -1,33 +0,0 @@
-class Config(object):
-    DEBUG = False
-    TESTING = False
-    SQLALCHEMY_DATABASE_URI = "sqlite:///data.db"
-    DUMMY_TRANSFER_UID = 'NOTAMEMBER'
-
-    SPACEAUTH_CONSUMER_KEY = 'kasownik'
-    SPACEAUTH_CONSUMER_SECRET = 'changeme'
-
-    SECRET_KEY = 'changeme'
-
-    LDAP_URI = 'ldap://ldap.hackerspace.pl'
-    LDAP_BIND_DN = 'cn=fascist,ou=Services,dc=hackerspace,dc=pl'
-    LDAP_BIND_PASSWORD = 'changeme'
-
-    LDAP_USER_FILTER = '(objectClass=hsMember)'
-    LDAP_USER_BASE = 'ou=People,dc=hackerspace,dc=pl'
-    LDAP_CA_PATH = '/etc/ssl/certs/ca-certificates.crt'
-
-    CACHE_TYPE = 'null'
-    CACHE_NO_NULL_WARNING = True
-
-    MEMBERSHIP_FEES = {
-        'starving': 75,
-        'fatty': 150,
-    }
-
-class DevelopmentConfig(Config):
-    DEBUG = True
-    DISABLE_LDAP = True
-
-class CurrentConfig(DevelopmentConfig):
-    pass

web/requirements.txt

@@ -1,7 +1,7 @@
 amqp==1.4.6
 astroid==1.6.1
 backports.functools-lru-cache==1.5
-beautifulsoup4==4.3.2
+beautifulsoup4==4.12.3
 billiard==3.3.0.19
 blinker==1.4
 cached-property==0.1.5
@@ -11,13 +11,14 @@ chardet==3.0.4
 click==6.7
 configparser==3.5.0
 enum34==1.1.6
+environs==11.0.0
 Flask==0.12.2
 Flask-Caching==1.3.3
 Flask-Gravatar==0.4.2
 Flask-Login==0.4.1
 Flask-OAuthlib==0.9.4
 Flask-Prometheus==0.0.1
-Flask-SpaceAuth @ git+https://code.hackerspace.pl/informatic/flask-spaceauth@0f997ab8fa307f7eabacf27e5f19fc1c715a69f3
+Flask-SpaceAuth @ https://code.hackerspace.pl/informatic/flask-spaceauth/archive/v0.2.0.tar.gz
 Flask-SQLAlchemy==2.1
 Flask-WTF==0.10.3
 idna==2.10
@@ -27,14 +28,17 @@ Jinja2==2.11.3
 lazy-object-proxy==1.3.1
 Mako==1.0.0
 MarkupSafe==1.1.0
+marshmallow==3.21.3
 mccabe==0.6.1
 oauthlib==2.0.7
+packaging==24.1
 prometheus-client==0.1.1
 prometheus-flask-exporter==0.8.0
-psycopg2==2.9.4
+psycopg2-binary==2.9.4
 pyasn1==0.4.8
 pyasn1-modules==0.2.8
 pylint==1.8.2
+python-dotenv==1.0.1
 python-ldap==3.3.1
 python-memcached==1.59
 pytz==2014.10
@@ -42,10 +46,12 @@ requests==2.24.0
 requests-oauthlib==0.8.0
 singledispatch==3.4.0.3
 six==1.11.0
+soupsieve==2.5
 SQLAlchemy==1.3.24
 sqlparse==0.4.1
 sqltap==0.3.5
 urllib3==1.25.10
+uWSGI==2.0.26
 Werkzeug==0.16.1
 wrapt==1.10.11
 WTForms==2.0.1

web/webapp/__init__.py

@@ -25,7 +25,6 @@
 import os
 from functools import wraps
 
-import memcache
 import requests
 import sqltap.wsgi
 import click
@@ -40,13 +39,19 @@ from flask_gravatar import Gravatar
 from spaceauth import SpaceAuth
 
 app = Flask(__name__)
-app.config.from_object("config.CurrentConfig")
+app.config.from_object("config")
 
 auth = SpaceAuth()
 db = SQLAlchemy()
 cache = Cache()
-gravatar = Gravatar(size=256, rating='g', default='retro',
-                    force_default=False, use_ssl=True, base_url=None)
+gravatar = Gravatar(
+    size=256,
+    rating="g",
+    default="retro",
+    force_default=False,
+    use_ssl=True,
+    base_url=None,
+)
 
 # TODO unsubscribe me from life
 cache_enabled = False
@@ -57,16 +62,20 @@ import webapp.models  # noqa
 from webapp.authutils import AnonymousUser, User
 
 auth.anonymous_user = AnonymousUser
+
+
 @auth.user_loader
 def load_user(username, profile=None):
     return User(username)
 
+
 def admin_required(func):
     @wraps(func)
     def wrapper(*args, **kwargs):
         if not current_user.is_admin:
             return auth.unauthorized()
         return func(*args, **kwargs)
+
     return wrapper
 
 
@@ -77,10 +86,10 @@ class DecimalEncoder(json.JSONEncoder):
             # but that would mean a yield on the line with super(...),
             # which wouldn't work (see my comment below), so...
             return str(o)
-            #return (str(o) for o in [o])
+            # return (str(o) for o in [o])
         elif isinstance(o, datetime.datetime):
             return str(o)
-            #return (str(o) for o in [o])
+            # return (str(o) for o in [o])
         return super(DecimalEncoder, self).default(o)
 
 
@@ -95,40 +104,43 @@ def create_app():
         app.wsgi_app = sqltap.wsgi.SQLTapMiddleware(app.wsgi_app)
 
     # Setup prometheus metrics
-    if app.config.get('PROMETHEUS_DIR'):
+    if app.config.get("PROMETHEUS_DIR"):
         # This needs to be set before importing prometheus_client
-        os.environ['prometheus_multiproc_dir'] = app.config['PROMETHEUS_DIR']
+        os.environ["prometheus_multiproc_dir"] = app.config["PROMETHEUS_DIR"]
 
         # FIXME: we could expose this somehow
         from prometheus_flask_exporter.multiprocess import UWsgiPrometheusMetrics
-        metrics = UWsgiPrometheusMetrics(group_by='url_rule')
+
+        metrics = UWsgiPrometheusMetrics(group_by="url_rule")
 
         metrics.init_app(app)
-        #metrics.register_endpoint('/varz', app)
+        # metrics.register_endpoint('/varz', app)
 
     # Register blueprints
     import webapp.views
     import webapp.admin
     import webapp.api
+
     app.register_blueprint(webapp.admin.bp)
     app.register_blueprint(webapp.api.bp)
 
     # Custom filters
-    @app.template_filter('inflect')
+    @app.template_filter("inflect")
     def inflect(v, one, two, five):
         num = abs(v)
 
         if num == 0:
-            return '%d %s' % (v, five)
+            return "%d %s" % (v, five)
         elif num == 1:
-            return '%d %s' % (v, one)
+            return "%d %s" % (v, one)
         elif num <= 4:
-            return '%d %s' % (v, two)
+            return "%d %s" % (v, two)
 
-        return '%d %s' % (v, five)
+        return "%d %s" % (v, five)
 
     # Custom CLI commands
     import webapp.commands
+
     app.cli.commands.update(webapp.commands.group.commands)
 
     app.json_encoder = DecimalEncoder

web/webapp/directory.py

@@ -34,6 +34,8 @@ from webapp import mc, cache_enabled, app
 
 def connect():
     c = ldap.initialize(app.config['LDAP_URI'])
+    c.set_option(ldap.OPT_X_TLS_CACERTFILE, app.config['LDAP_CA_PATH'])
+    c.set_option(ldap.OPT_X_TLS_NEWCTX, 0)
     c.start_tls_s()
     c.simple_bind_s(app.config['LDAP_BIND_DN'],
                     app.config['LDAP_BIND_PASSWORD'])