first analysis running detached with status updates through redis
Clemens Klug
parent
7d93b5f6fd
commit
01e2433b8b
|
|
@ -49,6 +49,7 @@ class KMLRender(Render):
|
|||
result_types = [LocationAnalyzer]
|
||||
|
||||
def render(self, results: List[Result], name=None):
|
||||
files = []
|
||||
for result in self.filter(results):
|
||||
times = ["<when>{time}</when>".format(time=format_time(entry["timestamp"])) for entry in result.get()]
|
||||
coords = [
|
||||
|
|
@ -62,6 +63,8 @@ class KMLRender(Render):
|
|||
print(filename)
|
||||
with open(filename, "w") as out:
|
||||
out.write(KML_PATTERN.format(name=str(result.name), coordinates="\n".join(coords), when="\n".join(times)))
|
||||
files.append(filename)
|
||||
return files
|
||||
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -35,6 +35,8 @@ class LogSettings:
|
|||
self.custom = json_dict['custom']
|
||||
if "source" in json_dict:
|
||||
self.source = load_source(json_dict['source'])
|
||||
if "render" in json_dict:
|
||||
self.render = json_dict['render']
|
||||
|
||||
def __repr__(self):
|
||||
return str({
|
||||
|
|
|
|||
|
|
@ -75,8 +75,10 @@ class BiogamesClient(Client):
|
|||
def login(self) -> bool:
|
||||
csrf_request = self.get(self.list_url)
|
||||
if not csrf_request.ok:
|
||||
raise ConnectionError("Unable to obtain CSRF token (" + str(csrf_request) + ")")
|
||||
self.cookies['csrftoken'] = csrf_request.cookies['csrftoken']
|
||||
log.exception(ConnectionError("Unable to obtain CSRF token (" + str(csrf_request) + ")"))
|
||||
return False
|
||||
if not 'csrftoken' in self.cookies:
|
||||
self.cookies['csrftoken'] = csrf_request.cookies['csrftoken']
|
||||
login_payload = {
|
||||
'username': self.config['username'],
|
||||
'password': self.config['password'],
|
||||
|
|
@ -85,7 +87,8 @@ class BiogamesClient(Client):
|
|||
}
|
||||
login = self.post(self.login_url, json.dumps(login_payload))
|
||||
if not login.ok:
|
||||
raise ConnectionError("Unable to authenticate", login, login.text)
|
||||
log.exception(ConnectionError("Unable to authenticate", login, login.text))
|
||||
return False
|
||||
self.cookies['sessionid'] = login.cookies['sessionid']
|
||||
print(self.cookies)
|
||||
return True
|
||||
|
|
|
|||
|
|
@ -19,12 +19,6 @@ services:
|
|||
- "traefik.docker.network=traefik_net"
|
||||
- "traefik.url.frontend.rule=Host:select.ma.potato.kinf.wiai.uni-bamberg.de"
|
||||
|
||||
|
||||
redis:
|
||||
image: redis:4-alpine
|
||||
|
||||
|
||||
|
||||
celery:
|
||||
image: docker.clkl.de/ma/celery:0.1
|
||||
build: .
|
||||
|
|
@ -32,8 +26,27 @@ services:
|
|||
- PYTHONPATH=/app
|
||||
volumes:
|
||||
- ./:/app
|
||||
working_dir: /app/tasks
|
||||
command: celery -A tasks worker --loglevel=info
|
||||
- ./data/results:/data/results
|
||||
working_dir: /app
|
||||
command: celery -A tasks.tasks worker --loglevel=info
|
||||
|
||||
|
||||
redis:
|
||||
image: redis:4-alpine
|
||||
volumes:
|
||||
- ./data/redis:/data
|
||||
command: redis-server --appendonly yes
|
||||
|
||||
nginx:
|
||||
image: nginx:1.13-alpine
|
||||
volumes:
|
||||
- ./data/results:/usr/share/nginx/html:ro
|
||||
labels:
|
||||
- "traefik.enable=true"
|
||||
- "traefik.port=80"
|
||||
- "traefik.docker.network=traefik_net"
|
||||
- "traefik.url.frontend.rule=Host:results.ma.potato.kinf.wiai.uni-bamberg.de"
|
||||
|
||||
|
||||
networks:
|
||||
traefik_net:
|
||||
|
|
|
|||
|
|
@ -4,3 +4,6 @@ body {
|
|||
#data{
|
||||
display: none;
|
||||
}
|
||||
li{
|
||||
list-style-type: none;
|
||||
}
|
||||
|
|
@ -0,0 +1,63 @@
|
|||
KML = """{
|
||||
"logFormat": "zip",
|
||||
"entryType": "@class",
|
||||
"spatials": [
|
||||
"de.findevielfalt.games.game2.instance.log.entry.LogEntryLocation"
|
||||
],
|
||||
"actions": [],
|
||||
"boards": [
|
||||
"de.findevielfalt.games.game2.instance.log.entry.ShowBoardLogEntry"
|
||||
],
|
||||
"analyzers": {
|
||||
"analysis.analyzers": [
|
||||
"BiogamesCategorizer",
|
||||
"LocationAnalyzer"
|
||||
]
|
||||
},
|
||||
"sequences": {
|
||||
"start": "de.findevielfalt.games.game2.instance.log.entry.LogEntryCache",
|
||||
"end": {
|
||||
"@class": "de.findevielfalt.games.game2.instance.log.entry.LogEntryInstanceAction",
|
||||
"action.@class": "de.findevielfalt.games.game2.instance.action.CacheEnableAction"
|
||||
}
|
||||
},
|
||||
"custom": {
|
||||
"simulation_rounds": [
|
||||
"de.findevielfalt.games.game2.instance.log.entry.LogEntryQuestion"
|
||||
],
|
||||
"simu_data": [
|
||||
"de.findevielfalt.games.game2.instance.data.sequence.simulation.SimulationBoardData"
|
||||
],
|
||||
"instance_start": "de.findevielfalt.games.game2.instance.log.entry.LogEntryStartInstance",
|
||||
"instance_id": "instance_id",
|
||||
"instance_config_id": "config.@id",
|
||||
"sequences2": {
|
||||
"id_field": "sequence_id",
|
||||
"start": {
|
||||
"@class": "de.findevielfalt.games.game2.instance.log.entry.ShowSequenceLogEntry",
|
||||
"action": "START"
|
||||
},
|
||||
"end": {
|
||||
"@class": "de.findevielfalt.games.game2.instance.log.entry.ShowSequenceLogEntry",
|
||||
"action": "PAUSE"
|
||||
}
|
||||
},
|
||||
"coordinates": "location.coordinates",
|
||||
"metadata": {
|
||||
"timestamp": "timestamp",
|
||||
"gamefield": "instance_id",
|
||||
"user": "player_group_name"
|
||||
}
|
||||
},
|
||||
"source": {
|
||||
"type": "Biogames",
|
||||
"username": "ba",
|
||||
"password": "853451",
|
||||
"host": "http://biogames.potato.kinf.wiai.uni-bamberg.de"
|
||||
},
|
||||
"render": [
|
||||
"KMLRender"
|
||||
]
|
||||
}"""
|
||||
|
||||
CONFIGS = {"KML": KML}#TODO
|
||||
|
|
@ -1,19 +1,41 @@
|
|||
{% extends "base.html" %}
|
||||
{% block body %}
|
||||
<form action="/start" method="post">
|
||||
<div id="data"> {{logs}} </div>
|
||||
<div id="data"> {{logs}}</div>
|
||||
<ul>
|
||||
{% for log in logs %}
|
||||
<li>
|
||||
<input type="checkbox" name="logs" value="{{log['@id']}}">
|
||||
{{log.start_date}}: {{log.player_group_name}} (→{{log.end_date}})
|
||||
</li>
|
||||
<li>
|
||||
<input type="checkbox" name="logs" value="{{log['@id']}}">
|
||||
{{log.start_date}}: {{log.player_group_name}} (→{{log.end_date}})
|
||||
</li>
|
||||
<!--{{log}}-->
|
||||
{% endfor %}
|
||||
</ul>
|
||||
<input type="checkbox" id="safety"><label for="safety">Confirm selection</label>
|
||||
<!--input type="checkbox" id="safety"><label for="safety">Confirm selection</label-->
|
||||
<input type="text" id="name" maxlength="128" placeholder="name" name="name"/><br>
|
||||
<select name="config">
|
||||
{% for config in configs %}
|
||||
<option>{{config}}</option>
|
||||
{% endfor %}
|
||||
</select>
|
||||
<input type="submit">
|
||||
|
||||
</form>
|
||||
|
||||
<div id="results">
|
||||
<ul>
|
||||
{% for job in jobs %}
|
||||
<li> {{jobs[job].status}}: "{{job}}":
|
||||
<ul>
|
||||
{% for r in jobs[job].results %}
|
||||
<li>{{r}}</li>
|
||||
{% endfor %}
|
||||
</ul>
|
||||
|
||||
|
||||
</li>
|
||||
{% endfor %}
|
||||
</ul>
|
||||
</div>
|
||||
|
||||
{% endblock %}
|
||||
|
|
@ -1,14 +1,23 @@
|
|||
import json
|
||||
import logging
|
||||
import typing
|
||||
import uuid
|
||||
|
||||
import time
|
||||
|
||||
from clients.webclients import Client, CLIENTS
|
||||
|
||||
from flask import Flask, render_template, request, redirect, session
|
||||
|
||||
from tasks import tasks
|
||||
from selector.temp_config import CONFIGS
|
||||
|
||||
BIOGAMES_HOST = "http://biogames.potato.kinf.wiai.uni-bamberg.de"
|
||||
|
||||
app = Flask(__name__)
|
||||
clients: typing.Dict[str, Client] = {}
|
||||
log: logging.Logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
@app.route("/")
|
||||
def index():
|
||||
|
|
@ -24,6 +33,7 @@ def login():
|
|||
if client.login():
|
||||
session['logged_in'] = True
|
||||
session['uid'] = str(uuid.uuid4())
|
||||
session['username'] = request.form['username']
|
||||
session['cookies'] = client.cookies
|
||||
session['game'] = game
|
||||
session['host'] = BIOGAMES_HOST
|
||||
|
|
@ -38,16 +48,38 @@ def games():
|
|||
return redirect("/")
|
||||
if session['logged_in'] and not session['uid'] in clients:
|
||||
clients[session['uid']] = CLIENTS[session['game']](host=session['host'], **session['cookies'])
|
||||
return render_template("games.html", logs=clients[session['uid']].list())
|
||||
job_status = json.loads(tasks.redis.get(session['username']))
|
||||
return render_template("games.html", logs=clients[session['uid']].list(), configs=CONFIGS, jobs=job_status)
|
||||
|
||||
|
||||
@app.route("/start", methods=['POST'])
|
||||
def start():
|
||||
pass
|
||||
print(str(request.form['logs']))
|
||||
status = {
|
||||
"status": "PENDING",
|
||||
"submit": time.strftime("%c"),
|
||||
"log_ids": request.form.getlist('logs'),
|
||||
"config": request.form['config'],
|
||||
}
|
||||
params = {
|
||||
"log_ids": request.form.getlist('logs'),
|
||||
"config": CONFIGS[request.form['config']],
|
||||
"username": session['username'],
|
||||
"cookies": session['cookies'],
|
||||
"host": session['host'],
|
||||
"clientName": session['game'],
|
||||
"name": request.form['name'],
|
||||
}
|
||||
tasks.status_update(session['username'], request.form['name'], status)
|
||||
tasks.analyze.delay(**params)
|
||||
return redirect("/games")
|
||||
|
||||
|
||||
@app.route("/status")
|
||||
def status():
|
||||
pass
|
||||
return json.dumps(json.loads(tasks.redis.get(session['username'])), indent=2)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
app.config.update({"SECRET_KEY":"59765798-2784-11e8-8d05-db4d6f6606c9"})
|
||||
app.config.update({"SECRET_KEY": "59765798-2784-11e8-8d05-db4d6f6606c9"})
|
||||
app.run(host="0.0.0.0", debug=True)
|
||||
|
|
|
|||
|
|
@ -0,0 +1,65 @@
|
|||
from .tasks import add, analyze
|
||||
|
||||
__log__ = ["/app/data/008cad400ab848f729913d034a.zip"]
|
||||
|
||||
__config__ = """{
|
||||
"logFormat": "zip",
|
||||
"entryType": "@class",
|
||||
"spatials": [
|
||||
"de.findevielfalt.games.game2.instance.log.entry.LogEntryLocation"
|
||||
],
|
||||
"actions": [
|
||||
"...QuestionAnswerEvent",
|
||||
"...SimuAnswerEvent"
|
||||
],
|
||||
"boards": [
|
||||
"de.findevielfalt.games.game2.instance.log.entry.ShowBoardLogEntry"
|
||||
],
|
||||
"analyzers": {
|
||||
"analysis.analyzers": [
|
||||
"BiogamesCategorizer",
|
||||
"LocationAnalyzer"
|
||||
]
|
||||
},
|
||||
"sequences": {
|
||||
"start": "de.findevielfalt.games.game2.instance.log.entry.LogEntryCache",
|
||||
"end": {
|
||||
"@class": "de.findevielfalt.games.game2.instance.log.entry.LogEntryInstanceAction",
|
||||
"action.@class": "de.findevielfalt.games.game2.instance.action.CacheEnableAction"
|
||||
}
|
||||
},
|
||||
"custom": {
|
||||
"simulation_rounds": [
|
||||
"de.findevielfalt.games.game2.instance.log.entry.LogEntryQuestion"
|
||||
],
|
||||
"simu_data": [
|
||||
"de.findevielfalt.games.game2.instance.data.sequence.simulation.SimulationBoardData"
|
||||
],
|
||||
"instance_start": "de.findevielfalt.games.game2.instance.log.entry.LogEntryStartInstance",
|
||||
"instance_id": "instance_id",
|
||||
"instance_config_id": "config.@id",
|
||||
"sequences2": {
|
||||
"id_field": "sequence_id",
|
||||
"start": {
|
||||
"@class": "de.findevielfalt.games.game2.instance.log.entry.ShowSequenceLogEntry",
|
||||
"action": "START"
|
||||
},
|
||||
"end": {
|
||||
"@class": "de.findevielfalt.games.game2.instance.log.entry.ShowSequenceLogEntry",
|
||||
"action": "PAUSE"
|
||||
}
|
||||
},
|
||||
"coordinates": "location.coordinates",
|
||||
"metadata": {
|
||||
"timestamp": "timestamp",
|
||||
"gamefield": "instance_id",
|
||||
"user": "player_group_name"
|
||||
}
|
||||
},
|
||||
"source": {
|
||||
"type": "Biogames",
|
||||
"username": "ba",
|
||||
"password": "853451",
|
||||
"host": "http://biogames.potato.kinf.wiai.uni-bamberg.de"
|
||||
}
|
||||
}"""
|
||||
|
|
@ -1,14 +1,81 @@
|
|||
import json
|
||||
import logging
|
||||
import shutil
|
||||
import uuid
|
||||
import os.path
|
||||
import os
|
||||
|
||||
import redis as redis_lib
|
||||
import time
|
||||
from celery import Celery
|
||||
from analysis import log_analyzer as la
|
||||
from analysis.analyzers import KMLRender
|
||||
from clients.webclients import CLIENTS
|
||||
|
||||
FLASK_DB = 1
|
||||
REDIS_HOST = "redis"
|
||||
DATA_PATH = "/app/data/results_test/"
|
||||
|
||||
RENDERERS = { # TODO
|
||||
"KMLRender": KMLRender
|
||||
}
|
||||
|
||||
app = Celery('tasks', backend='redis://redis', broker='redis://redis')
|
||||
redis = redis_lib.StrictRedis(host=REDIS_HOST, db=FLASK_DB)
|
||||
log: logging.Logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def update_status(username, name, state, **kwargs):
|
||||
status = json.loads(redis.get(username))
|
||||
status[name][state[0]] = time.strftime("%c")
|
||||
status[name]['status'] = state[1]
|
||||
for i in kwargs:
|
||||
status[name][i] = kwargs[i]
|
||||
redis.set(username, json.dumps(status))
|
||||
|
||||
|
||||
@app.task
|
||||
def add(x, y):
|
||||
return x + y
|
||||
def analyze(config, log_ids, **kwargs):
|
||||
update_status(kwargs['username'], kwargs['name'], ('load', 'LOADING'))
|
||||
|
||||
log.info("start analysis")
|
||||
client = CLIENTS[kwargs['clientName']](host=kwargs['host'], **kwargs['cookies'])
|
||||
logs = client.list()
|
||||
id_urls = {str(x['@id']): x['file_url'] for x in logs}
|
||||
urls = [id_urls[i] for i in log_ids]
|
||||
tmpdir = client.download_files(urls)
|
||||
log.info(tmpdir.name, list(os.scandir(tmpdir.name)))
|
||||
|
||||
update_status(kwargs['username'], kwargs['name'], ('start', 'RUNNING'))
|
||||
|
||||
@app.task
|
||||
def analyze(config, log_ids):
|
||||
settings = la.parse_settings(config)
|
||||
store = la.run_analysis(log_ids, settings, la.LOADERS)
|
||||
store = la.run_analysis([p.path for p in os.scandir(tmpdir.name)], settings, la.LOADERS)
|
||||
render = RENDERERS[settings.render[0]]() # TODO
|
||||
files = render.render(store.get_all())
|
||||
|
||||
uid = str(uuid.uuid4())
|
||||
results = []
|
||||
log.error(files)
|
||||
os.mkdir(os.path.join(DATA_PATH, uid))
|
||||
for file in files:
|
||||
try:
|
||||
head, tail = os.path.split(file)
|
||||
target = os.path.join(DATA_PATH, uid, tail)
|
||||
shutil.move(file, target)
|
||||
results.append(target)
|
||||
except FileNotFoundError as e:
|
||||
log.exception(e)
|
||||
tmpdir.cleanup()
|
||||
|
||||
update_status(kwargs['username'], kwargs['name'], ('done', 'FINISHED'), results=results)
|
||||
|
||||
|
||||
def status_update(key, status_key, status):
|
||||
record = redis.get(key)
|
||||
if not record:
|
||||
redis.set(key, json.dumps({status_key: status}))
|
||||
else:
|
||||
data = json.loads(record)
|
||||
data[status_key] = status
|
||||
redis.set(key, json.dumps(data))
|
||||
redis.save()
|
||||
|
|
|
|||
Loading…
Reference in New Issue