fetch: add environment variable interpolation

Dockerfile

@@ -2,13 +2,16 @@ FROM python:3.9-slim
 
 RUN apt-get update && apt-get install --no-install-recommends -y build-essential libsasl2-dev libldap2-dev
 
-ADD requirements.txt .
+ADD web/requirements.txt .
+
 # Downgrade setuptools to fix anyjson dependency
 RUN pip install setuptools~=57.5.0 && \
     pip install -r requirements.txt
 
-WORKDIR /usr/src
-ADD . .
+WORKDIR /usr/src/web
+
+ADD web /usr/src/web
+ADD fetch /usr/src/fetch
 
 STOPSIGNAL SIGINT
 CMD ["uwsgi", "--http-socket", "0.0.0.0:5000", "--plugins", "python3", "--wsgi", "webapp.wsgi:app", "--threads", "10", "--master"]

docker-compose.yml

@@ -2,11 +2,12 @@ version: "3.8"
 
 services:
   kasownik-web:
-    build: web
+    build: .
     ports:
       - 5000:5000
     volumes:
-      - ./web:/usr/src
+      - ./web:/usr/src/web
+      - ./fetch:/usr/src/fetch
     environment:
       - SPACEAUTH_CONSUMER_KEY
       - SPACEAUTH_CONSUMER_SECRET

fetch/banking-pekaobiznes.py

@@ -371,7 +371,7 @@ def release(fn):
         sys.exit(3)
 
 parser = argparse.ArgumentParser()
-parser.add_argument('--config', help="Load configuration file", default="config.ini")
+parser.add_argument('--config', help="Load configuration file")
 parser.add_argument('-n', '--no-action', action="store_true", help='do not commit any database changes')
 parser.add_argument('-c', '--cached', action="store_true", help='use cached data (test)')
 parser.add_argument('-l', '--load', action='append', help='process specified files (test)')
@@ -382,8 +382,19 @@ parser.add_argument('--print-schema', action="store_true", help='print table sch
 if __name__ == "__main__":
     args = parser.parse_args()
 
-    config = configparser.ConfigParser()
-    config.read(args.config)
+    config = configparser.ConfigParser(defaults=os.environ, interpolation=configparser.ExtendedInterpolation())
+    config.read_dict({
+        'logging': {
+            'level': 'INFO',
+        },
+        'general': {
+            'cache_dir': 'cache',
+            'lockfile': 'lockfile',
+        },
+    })
+
+    if args.config:
+        config.read(args.config)
 
     logging.basicConfig(level=config['logging']['level'], format=config['logging'].get('format', '%(asctime)s [%(levelname)s] %(name)s: %(message)s'))
     logging.getLogger('chardet').setLevel(logging.WARN)

web/requirements.txt

@@ -1,7 +1,7 @@
 amqp==1.4.6
 astroid==1.6.1
 backports.functools-lru-cache==1.5
-beautifulsoup4==4.3.2
+beautifulsoup4==4.12.3
 billiard==3.3.0.19
 blinker==1.4
 cached-property==0.1.5
@@ -46,6 +46,7 @@ requests==2.24.0
 requests-oauthlib==0.8.0
 singledispatch==3.4.0.3
 six==1.11.0
+soupsieve==2.5
 SQLAlchemy==1.3.24
 sqlparse==0.4.1
 sqltap==0.3.5