From 36ff8fd0bed1da2fde527ae197bb606ae66ca47c Mon Sep 17 00:00:00 2001 From: agp8x Date: Wed, 16 Aug 2017 15:26:44 +0200 Subject: [PATCH] work on result rendering * add tracks + heat map * copy result lists instead of using just a reference * introduce proper logging --- analyzers/__init__.py | 23 ++++++++--- analyzers/analyzer/__init__.py | 7 +++- analyzers/analyzer/biogames.py | 21 ++++++++++ analyzers/analyzer/default.py | 18 +++------ analyzers/analyzer/locomotion.py | 2 +- analyzers/render/biogames.py | 35 ++++++++++++++++- analyzers/render/default.py | 35 +++++++++++++++++ biogames2.json | 10 +++-- log_analyzer.py | 13 +++++-- sample.json | 1 - static/heatmap.html | 46 ++++++++++++++++++++++ static/trackmap.html | 66 ++++++++++++++++++++++++++++++++ 12 files changed, 249 insertions(+), 28 deletions(-) delete mode 100644 sample.json create mode 100644 static/heatmap.html create mode 100644 static/trackmap.html diff --git a/analyzers/__init__.py b/analyzers/__init__.py index 1bcc8ca..63014aa 100644 --- a/analyzers/__init__.py +++ b/analyzers/__init__.py @@ -1,15 +1,15 @@ from typing import List from .analyzer import Analyzer, Result -from .analyzer.biogames import BoardDurationAnalyzer, SimulationRoundsAnalyzer +from .analyzer.biogames import BoardDurationAnalyzer, SimulationRoundsAnalyzer, ActivationSequenceAnalyzer from .analyzer.default import LogEntryCountAnalyzer, LocationAnalyzer, LogEntrySequenceAnalyzer, ActionSequenceAnalyzer from .analyzer.locomotion import LocomotionActionAnalyzer, CacheSequenceAnalyzer from .analyzer.mask import MaskSpatials from .render import Render -from .render.default import PrintRender, JSONRender +from .render.default import PrintRender, JSONRender, TrackRender, HeatMapRender from .render.locomotion import LocomotionActionRelativeRender, LocomotionActionAbsoluteRender, \ LocomotionActionRatioRender -from .render.biogames import SimulationRoundsRender +from .render.biogames import SimulationRoundsRender, BoardDurationHistRender, BoardDurationBoxRender __FALLBACK__ = PrintRender __MAPPING__ = { @@ -23,7 +23,18 @@ __MAPPING__ = { SimulationRoundsAnalyzer: [ JSONRender, SimulationRoundsRender, - ] + ], + BoardDurationAnalyzer: [ + BoardDurationHistRender, + BoardDurationBoxRender, + ], + ActivationSequenceAnalyzer: [ + JSONRender, + ], + LocationAnalyzer: [ + TrackRender, + #HeatMapRender, + ], } @@ -37,4 +48,6 @@ def render(cls: type, results: List[Result]): for r in get_renderer(cls): p = r() p.result_types.append(cls) - p.render(results) + rendered = p.render(results) + if rendered: + print(str(r)) diff --git a/analyzers/analyzer/__init__.py b/analyzers/analyzer/__init__.py index b8d8c60..e24b7bf 100644 --- a/analyzers/analyzer/__init__.py +++ b/analyzers/analyzer/__init__.py @@ -1,19 +1,24 @@ from analyzers.settings import LogSettings +import logging + +log = logging.getLogger(__name__) class Result: def __init__(self, analysis, result): self.result = result self.__analysis__ = analysis + log.debug("set" + str(len(self.result))) def analysis(self): return self.__analysis__ def get(self): + log.debug("get" + str(len(self.result))) return self.result def __repr__(self): - return "" + return "" class Analyzer: diff --git a/analyzers/analyzer/biogames.py b/analyzers/analyzer/biogames.py index 8d89e83..d3db374 100644 --- a/analyzers/analyzer/biogames.py +++ b/analyzers/analyzer/biogames.py @@ -1,7 +1,10 @@ +import logging from collections import defaultdict from . import Result, LogSettings, Analyzer +logger = logging.getLogger(__name__) + class BoardDurationAnalyzer(Analyzer): """ @@ -61,3 +64,21 @@ class SimulationRoundsAnalyzer(Analyzer): simu_id = entry['answers']["@id"] self.store[simu_id] += 1 return False + + +class ActivationSequenceAnalyzer(Analyzer): + __name__ = "ActivationSequence" + + def __init__(self, settings: LogSettings): + super().__init__(settings) + self.store = [] + + def result(self) -> Result: + return Result(type(self), self.store) + + def process(self, entry: dict) -> bool: + if entry[self.settings.type_field] in self.settings.sequences['start']: + if entry['cache']: + self.store.append(entry['cache']['@id']) + else: + logger.error("null cache") diff --git a/analyzers/analyzer/default.py b/analyzers/analyzer/default.py index 163d660..87a3a68 100644 --- a/analyzers/analyzer/default.py +++ b/analyzers/analyzer/default.py @@ -1,4 +1,4 @@ -import json +import logging from collections import defaultdict from . import Result, LogSettings, Analyzer @@ -10,25 +10,19 @@ class LocationAnalyzer(Analyzer): """ entries = [] __name__ = "Location" - - class Formats: - geojson = 0 + log = logging.getLogger(__name__) def __init__(self, settings: LogSettings): super().__init__(settings) def result(self) -> Result: - # return self.entries - return Result(type(self), self.entries) - - def render(self, format: int = Formats.geojson): - if format is self.Formats.geojson: - return json.dumps([entry['location']['coordinates'] for entry in self.entries]) - raise NotImplementedError() + self.log.debug(len(self.entries)) + return Result(type(self), list(self.entries)) def process(self, entry: dict) -> bool: if entry[self.settings.type_field] in self.settings.spatials: self.entries.append(entry) + #self.log.debug(len(self.entries)) return False @@ -57,7 +51,7 @@ class LogEntrySequenceAnalyzer(Analyzer): __name__ = "LogEntrySequence" def result(self) -> Result: - return Result(type(self), self.store) + return Result(type(self), list(self.store)) def process(self, entry: dict) -> bool: entry_type = entry[self.settings.type_field] diff --git a/analyzers/analyzer/locomotion.py b/analyzers/analyzer/locomotion.py index 388a451..ff33525 100644 --- a/analyzers/analyzer/locomotion.py +++ b/analyzers/analyzer/locomotion.py @@ -87,7 +87,7 @@ class CacheSequenceAnalyzer(Analyzer): return False def result(self) -> Result: - return Result(type(self), self.store) + return Result(type(self), list(self.store)) def __init__(self, settings: LogSettings): super().__init__(settings) diff --git a/analyzers/render/biogames.py b/analyzers/render/biogames.py index dd9545a..797bb36 100644 --- a/analyzers/render/biogames.py +++ b/analyzers/render/biogames.py @@ -4,7 +4,7 @@ from typing import List, Tuple import matplotlib.pyplot as plt from . import Render -from .. import Result, SimulationRoundsAnalyzer +from .. import Result, SimulationRoundsAnalyzer, BoardDurationAnalyzer def plot(src_data: List[Tuple[str, List[int]]]): @@ -29,3 +29,36 @@ class SimulationRoundsRender(Render): plot(data_tuples) result_types = [SimulationRoundsAnalyzer] + +class BoardDurationHistRender(Render): + + result_types = [BoardDurationAnalyzer] + + def render(self, results: List[Result]): + data = [] + for result in self.filter(results): + session = result.get() + _data = [] + for board in session: + if "active" in board: + _data.append(board["active"]) + else: + _data.append(0) + data.append(_data) + n, bins, patches = plt.hist(data, log=True) + plt.show() + + +class BoardDurationBoxRender(Render): + result_types = [BoardDurationAnalyzer] + + def render(self, results: List[Result]): + data = defaultdict(list) + for result in self.filter(results): + get = result.get() + for board in get: + duration = board['active'] if 'active' in board else 0 + data[board['id']].append(duration) + data_tuples = [(key, data[key]) for key in sorted(data)] + data_tuples = sorted(data_tuples, key=lambda x: sum(x[1])) + plot(data_tuples) \ No newline at end of file diff --git a/analyzers/render/default.py b/analyzers/render/default.py index 0aaf858..e980b50 100644 --- a/analyzers/render/default.py +++ b/analyzers/render/default.py @@ -1,7 +1,11 @@ import json +import logging from typing import List from . import Render, Result +from .. import LocationAnalyzer + +log = logging.getLogger(__name__) class PrintRender(Render): @@ -12,3 +16,34 @@ class PrintRender(Render): class JSONRender(Render): def render(self, results: List[Result]): print(json.dumps([r.get() for r in self.filter(results)], indent=1)) + + +class TrackRender(Render): + result_types = [LocationAnalyzer] + + def render(self, results: List[Result]): + data = [] + log.debug(results) + for result in self.filter(results): + if len(result.get()) > 0: + data.append( + [[entry['location']['coordinates'][1], entry['location']['coordinates'][0]] for entry in + result.get()]) + dumps = json.dumps(data) + with open("track_data.js", "w") as out: + out.write("tracks=" + dumps + ";") + return dumps + + +class HeatMapRender(TrackRender): + weight = 0.01 + + def render(self, results: List[Result]): + raw = super(HeatMapRender, self).render(results) + data = [] + for session in json.loads(raw): + data += [(entry[0], entry[1], self.weight) for entry in session] + dumps = json.dumps(data) + with open('heat_data.js', 'w') as out: + out.write("coords = " + dumps + ";") + return dumps diff --git a/biogames2.json b/biogames2.json index 7e124e1..921368b 100644 --- a/biogames2.json +++ b/biogames2.json @@ -13,16 +13,18 @@ ], "analyzers": { "analyzers": [ - "LocationAnalyzer", + "LocationAnalyzer" + ] + }, + "disabled_analyzers":[ "LogEntryCountAnalyzer", "LogEntrySequenceAnalyzer", "ActionSequenceAnalyzer", "BoardDurationAnalyzer", "LocomotionActionAnalyzer", "CacheSequenceAnalyzer", - "SimulationRoundsAnalyzer" - ] - }, + "SimulationRoundsAnalyzer", + "ActivationSequenceAnalyzer"], "sequences": { "start": "de.findevielfalt.games.game2.instance.log.entry.LogEntryCache", "end": { diff --git a/log_analyzer.py b/log_analyzer.py index 4d137c4..4bc95e3 100644 --- a/log_analyzer.py +++ b/log_analyzer.py @@ -1,3 +1,4 @@ +import logging from load import LOADERS from typing import List from analyzers import get_renderer, Analyzer, render @@ -5,6 +6,10 @@ from analyzers.settings import LogSettings, load_settings import analyzers +logging.basicConfig(format='%(levelname)s %(name)s:%(message)s', level=logging.DEBUG) +logger = logging.getLogger(__name__) + + def process_log(log_id: str, settings: LogSettings) -> List[Analyzer]: logfile = "data/inst_{id}.{format}".format(id=log_id, format=settings.log_format) loader = LOADERS[settings.log_format]() @@ -28,7 +33,8 @@ if __name__ == '__main__': "20d4244719404ffab0ca386c76e4b112", "56d9b64144ab44e7b90bf766f3be32e3", "dc2cdc28ca074715b905e4aa5badff10", - "e32b16998440475b994ab46d481d3e0c", ] + "e32b16998440475b994ab46d481d3e0c", + ] log_ids = [ "34fecf49dbaca3401d745fb467", "44ea194de594cd8d63ac0314be", @@ -41,16 +47,17 @@ if __name__ == '__main__': "fe1331481f85560681f86827ec", "fec57041458e6cef98652df625", ] results = [] + #TODO: capture session ID, dict for log_id in log_ids: for analysis in process_log(log_id, settings): - print("* Result for " + analysis.name()) + logger.info("* Result for " + analysis.name()) # print(analysis.result()) # print(analysis.render()) results.append(analysis.result()) if False: for r in get_renderer(analyzers.LocomotionActionAnalyzer): r().render(results) - render(analyzers.SimulationRoundsAnalyzer, results) + render(analyzers.LocationAnalyzer, results) # for analyzers in analyzers: # if analyzers.name() in ["LogEntryCount", "ActionSequenceAnalyzer"]: diff --git a/sample.json b/sample.json deleted file mode 100644 index 0637a08..0000000 --- a/sample.json +++ /dev/null @@ -1 +0,0 @@ -[] \ No newline at end of file diff --git a/static/heatmap.html b/static/heatmap.html new file mode 100644 index 0000000..6d49b9a --- /dev/null +++ b/static/heatmap.html @@ -0,0 +1,46 @@ + + + + Leaflet.heat demo + + + + + + +
+ + + + + + + + + + diff --git a/static/trackmap.html b/static/trackmap.html new file mode 100644 index 0000000..2323a9a --- /dev/null +++ b/static/trackmap.html @@ -0,0 +1,66 @@ + + + + Leaflet.heat demo + + + + + + +
+ + + + + + + + + +