diff --git a/analysis/analyzers/__init__.py b/analysis/analyzers/__init__.py index 97d33b6..8446baf 100644 --- a/analysis/analyzers/__init__.py +++ b/analysis/analyzers/__init__.py @@ -3,18 +3,19 @@ from typing import List from .analyzer import Analyzer, Result from .analyzer.biogames import BoardDurationAnalyzer, SimulationRoundsAnalyzer, ActivationSequenceAnalyzer, \ BiogamesCategorizer, ActivityMapper, BiogamesStore, InstanceConfig, SimulationOrderAnalyzer, SimulationCategorizer, \ - SimulationFlagsAnalyzer + SimulationFlagsAnalyzer, GameField_InstanceCategorizer from .analyzer.default import LogEntryCountAnalyzer, LocationAnalyzer, LogEntrySequenceAnalyzer, ActionSequenceAnalyzer, \ - CategorizerStub, Store, ProgressAnalyzer, SimpleCategorizer + CategorizerStub, Store, ProgressAnalyzer, SimpleCategorizer, WhitelistAnalyzer from .analyzer.locomotion import LocomotionActionAnalyzer, CacheSequenceAnalyzer from .analyzer.mask import MaskSpatials from .render import Render from .render.biogames import SimulationRoundsRender, BoardDurationHistRender, BoardDurationBoxRender, \ ActivityMapperRender, StoreRender, SimulationOrderRender, SimulationGroupRender from .render.default import PrintRender, JSONRender, TrackRender, HeatMapRender, LogEntryCountAnalyzerPlot, \ - LogEntryCountCSV, KMLRender + LogEntryCountCSV, KMLRender, GeoJSON from .render.locomotion import LocomotionActionRelativeRender, LocomotionActionAbsoluteRender, \ LocomotionActionRatioRender +from .render.wip import get_all_data, plot_time_space_rel __FALLBACK__ = PrintRender __MAPPING__ = { diff --git a/analysis/analyzers/analyzer/biogames.py b/analysis/analyzers/analyzer/biogames.py index e87f21f..3edc637 100644 --- a/analysis/analyzers/analyzer/biogames.py +++ b/analysis/analyzers/analyzer/biogames.py @@ -133,6 +133,23 @@ class BiogamesCategorizer(CategorizerStub): # TODO: refactor return False +class GameField_InstanceCategorizer(CategorizerStub): # TODO: refactor + __name__ = "BiogamesCategorizer" + + def __init__(self, settings: LogSettings): + super().__init__(settings) + + def process(self, entry: dict) -> bool: + if self.key is "default": + if entry[self.settings.type_field] in self.settings.custom['instance_start']: + try: + self.key = json_path(entry, self.settings.custom['instance_config_id']) + "_" + entry[self.settings.custom['instance_id']] + "_" + str(entry["timestamp"]) + except KeyError as e: + print(entry) + raise e + return False + + class ActivityMapper(Analyzer): __name__ = "ActivityMapper" classes = { @@ -158,6 +175,7 @@ class ActivityMapper(Analyzer): self.last_board = {} self.last_board_type = "other" self.last_coordinate = None + self.last_timestamp = None self.tracks = [] self.track = None self.instance_config_id: str = None @@ -173,21 +191,28 @@ class ActivityMapper(Analyzer): if board["extra_data"]["activity_type"] == "simu": board["image"] = "simu.png" continue - local_file = download_board(board["board_id"], self.instance_config_id, board["sequence_id"], + try: + local_file = download_board(board["board_id"], self.instance_config_id, board["sequence_id"], self.settings.source) - if local_file: - board['image'] = local_file - else: + if local_file: + board['image'] = local_file + else: + raise ValueError + except Exception as e: board['image'] = "ERROR_FETCHING_FILE" logger.error("error downloading board! %s %s %s", self.instance_config_id, board["sequence_id"], board["board_id"]) + logger.exception(e) else: board["image"] = "map.png" store.add(Result(type(self), { - "instance": self.instance_config_id, - "track": self.tracks, - "boards": self.timeline, - "colors": self.colors, + "type": "FeatureCollection", + "features": self.tracks, + "properties": { + "instance": self.instance_config_id, + "boards": self.timeline, + "colors": self.colors, + }, })) def process(self, entry: dict) -> bool: @@ -242,18 +267,24 @@ class ActivityMapper(Analyzer): def add_location(self, entry): coordinates = json_path(entry, self.settings.custom['coordinates']) - self.track['coordinates'].append(coordinates) + self.track["geometry"]['coordinates'].append(coordinates) + self.track['properties']['coordTimes'].append(entry['timestamp']) #FIXME self.last_coordinate = coordinates + self.last_timestamp = entry['timestamp'] def add_track(self, **props): self.track['properties'].update(props) + if "activity_type" in self.track['properties'] and self.track['properties']['activity_type'] in self.colors: + if not "stroke" in self.track['properties']: + self.track['properties']['stroke'] = self.colors[self.track['properties']['activity_type']] self.tracks.append(self.track) self.track = self.new_track(props['end_timestamp']) if self.last_coordinate: - self.track['coordinates'].append(self.last_coordinate) + self.track["geometry"]['coordinates'].append(self.last_coordinate) + self.track['properties']['coordTimes'].append(self.last_timestamp) def new_track(self, timestamp): - return {"type": "LineString", "coordinates": [], "properties": {'start_timestamp': timestamp}} + return {"type": "Feature", "geometry": {"type": "LineString", "coordinates": []}, "properties": {'start_timestamp': timestamp, 'coordTimes': []}} class BiogamesStore(Store): diff --git a/analysis/analyzers/analyzer/default.py b/analysis/analyzers/analyzer/default.py index 745e9f0..c4233b9 100644 --- a/analysis/analyzers/analyzer/default.py +++ b/analysis/analyzers/analyzer/default.py @@ -5,6 +5,26 @@ from analysis.util import json_path from . import Result, LogSettings, Analyzer, ResultStore +class WhitelistAnalyzer(Analyzer): #TODO + __name__ = "WhiteList" + + def __init__(self, settings: LogSettings) -> None: + super().__init__(settings) + self.key = settings.custom["whitelist"]["key"] + self.filter_objs = settings.custom["whitelist"]["objs"] + self.valid_entries = settings.custom["whitelist"]["valid"] + self.blocked = False + + def result(self, store: ResultStore, name=None) -> None: + store.add(Result(type(self), {"blocked": self.blocked}, name=name)) + + def process(self, entry: dict) -> bool: + if entry[self.settings.type_field] in self.filter_objs: + if not json_path(entry, self.key) in self.valid_entries: + self.blocked = True + return self.blocked + + class LocationAnalyzer(Analyzer): """ store spatial log entries diff --git a/analysis/analyzers/analyzer/locomotion.py b/analysis/analyzers/analyzer/locomotion.py index 0eede5b..13af0b8 100644 --- a/analysis/analyzers/analyzer/locomotion.py +++ b/analysis/analyzers/analyzer/locomotion.py @@ -41,7 +41,7 @@ class LocomotionActionAnalyzer(Analyzer): self.current_cache = None self.last = None - def result(self, store: ResultStore) -> None: + def result(self, store: ResultStore, **kwargs) -> None: if self.last is not None: if self.current_cache is None: self.locomotion.append(self.last - self.cache_time) @@ -50,7 +50,14 @@ class LocomotionActionAnalyzer(Analyzer): self.last = None locomotion = sum(self.locomotion) action = sum(self.actions) + if action== 0: + action = 42 + print("Division by zero") #FIXME total = locomotion + action + if not self.last_timestamp: + self.last_timestamp = 0 + if not self.instance_start: + self.instance_start = 0 store.add(Result(type(self), { 'locomotion_sum': locomotion, 'action_sum': action, diff --git a/analysis/analyzers/render/biogames.py b/analysis/analyzers/render/biogames.py index 53c12c4..20b2cac 100644 --- a/analysis/analyzers/render/biogames.py +++ b/analysis/analyzers/render/biogames.py @@ -155,7 +155,7 @@ class ActivityMapperRender(Render): files = [] for result in self.filter(results): data = result.get() - path = os.path.join("/tmp", data['instance'] + "_" + str(name) + ".json") + path = os.path.join("/tmp", data["properties"]['instance'] + "_" + str(name) + ".json") with open(path, "w") as out: json.dump(data, out, indent=1) files.append(path) diff --git a/analysis/analyzers/render/default.py b/analysis/analyzers/render/default.py index 8904d21..8ee218f 100644 --- a/analysis/analyzers/render/default.py +++ b/analysis/analyzers/render/default.py @@ -61,13 +61,16 @@ class KMLRender(SpatialRender, Render): long=entry['location']['coordinates'][0]) for entry in result.get() ] - filename = str(result.name) + ".kml" + if name: + filename = str(name) + ".kml" + else: + filename = str(result.name) + ".kml" print(filename) with open(filename, "w") as out: out.write( KML_PATTERN.format(name=str(result.name), coordinates="\n".join(coords), when="\n".join(times))) - with open(filename + ".json", "w") as out: - json.dump(result.get(), out, indent=1) + #with open(filename + ".json", "w") as out: #FIXME: why am I here?? + # json.dump(result.get(), out, indent=1) files.append(filename) return files @@ -96,7 +99,10 @@ class GeoJSON(SpatialRender, Render): #print(location) coordinates.append(location["location"]["coordinates"]) times.append(location["timestamp"]) - filename = str(result.name) + ".geojson" + if name: + filename = str(name) + ".geojson" + else: + filename = str(result.name) + ".geojson" with open(filename, "w") as out: self.template["features"][0]["properties"] = {"times": times} self.template["features"][0]["geometry"]["coordinates"] = coordinates diff --git a/analysis/analyzers/render/wip.py b/analysis/analyzers/render/wip.py index 19249ec..71f7540 100644 --- a/analysis/analyzers/render/wip.py +++ b/analysis/analyzers/render/wip.py @@ -3,6 +3,7 @@ import json import numpy as np import analysis.analyzers +from analysis import analyzers from analysis.util.geo import calc_distance @@ -188,10 +189,11 @@ def get_all_data(store, sort=False, relative=True): if not log.analysis() == analyzers.ActivityMapper: continue result = defaultdict(lambda: defaultdict(lambda: 0)) - for i in log.get()['track']: - coords = i['coordinates'] + for i in log.get()['features']: + coords = i["geometry"]['coordinates'] + print(coords) if len(coords) > 1: - distance = calc_distance(coords) + distance = calc_distance(json.dumps(i["geometry"])) else: distance = 0.0 result["space"][i['properties']['activity_type']] += distance @@ -219,9 +221,9 @@ def get_all_data(store, sort=False, relative=True): value = percentage else: value = total - places[log.get()['instance']].append(value) + places[log.get()["properties"]['instance']].append(value) simus = defaultdict(lambda: 0) - for item in log.get()['boards']: + for item in log.get()["properties"]['boards']: if item["extra_data"]["activity_type"] == "simu": simus[item["board_id"]] += 1 simu_distribution[len(simus)] += 1 @@ -381,7 +383,7 @@ def plot_time_space_rel(combined, keys): print([CONFIG_NAMES[i] if i in CONFIG_NAMES else "---" for i in ids]) # plt.show() dpi = 200 - plt.savefig("speed2.png", dpi=dpi) + plt.savefig("speed2_2019.png", dpi=dpi) diff --git a/analysis/log_analyzer.py b/analysis/log_analyzer.py index 29cb9c6..b400fa3 100644 --- a/analysis/log_analyzer.py +++ b/analysis/log_analyzer.py @@ -109,7 +109,7 @@ if __name__ == '__main__': # plot_time_space_rel(combined, keys) plot_data(combined, wip.keys) - if True: + if False: def store(x): pass @@ -214,6 +214,21 @@ if __name__ == '__main__': print("\nsimplified\n") summary(stats_simple, stats_filtered_simple, "simplified") + if True: + settings: LogSettings = load_settings("time.json") + # log_ids = src_file("/app/log_data/oeb/oeb_2016_path") + log_ids = src_file("log_data/oeb/oeb_paths_host") + log_ids = src_file("/home/clemens/git/ma/test/src") + log_ids = src_file("/home/clemens/git/ma/project/log_data/neocartographer/index") + # log_ids = log_ids[0:10] + print(log_ids) + store: ResultStore = run_analysis(log_ids, settings, LOADERS, ResultStore(key_index=1)) + results = {} + for cat in store.get_categories(): + results[cat] = [result.get() for result in store.get_category(cat)] + with open("times_neo.json", "w") as out: + json.dump(results, out, indent=1) + #################### #for cat in store.get_categories(): # render(analyzers.ActivityMapper, store.get_category(cat), name=cat) diff --git a/analysis/util/download.py b/analysis/util/download.py index a70a81d..3ceea7b 100644 --- a/analysis/util/download.py +++ b/analysis/util/download.py @@ -6,7 +6,8 @@ from analysis.util import json_path logger = logging.getLogger(__name__) -def download_board(board_id, instance_config_id, sequence_id, source, path="/data/results/"): +#def download_board(board_id, instance_config_id, sequence_id, source, path="/data/results/"): #FIXME: I was changed +def download_board(board_id, instance_config_id, sequence_id, source, path="activity/data/results/"): local_file = os.path.join("static", instance_config_id, sequence_id, board_id) abs_path = os.path.join(path, local_file) if os.path.exists(abs_path): diff --git a/analysis/util/processing.py b/analysis/util/processing.py index 933897d..39b2d20 100644 --- a/analysis/util/processing.py +++ b/analysis/util/processing.py @@ -12,7 +12,7 @@ def process_log(logfile: str, settings: LogSettings, loaders) -> List[Analyzer]: try: loader.load(logfile) except BaseException as e: - raise RuntimeError(e) + raise RuntimeError(e, logfile) analyzers: List[Analyzer] = [] log.debug("build analyzers") for analyzer in settings.analyzers: @@ -28,7 +28,9 @@ def process_log(logfile: str, settings: LogSettings, loaders) -> List[Analyzer]: return analyzers -def run_analysis(log_ids: list, settings, loaders, result_store=ResultStore()): +def run_analysis(log_ids: list, settings, loaders, result_store=None): + if not result_store: + result_store = ResultStore() store: ResultStore = result_store for log_id in log_ids: log.info("LOG_ID: "+ str(log_id)) diff --git a/data/results/index.html b/data/results/index.html index 25464d6..e6d6d82 100644 --- a/data/results/index.html +++ b/data/results/index.html @@ -1,13 +1,7 @@ - - + src="https://code.jquery.com/jquery-3.2.1.min.js"> + + @@ -22,4 +16,4 @@ \ No newline at end of file + diff --git a/data/results/my.js b/data/results/my.js index 5c70a6f..078ffee 100644 --- a/data/results/my.js +++ b/data/results/my.js @@ -23,7 +23,7 @@ function loadData() { function styleTrack(feature) { var styles = {}; - styles.color = data.colors[feature.properties.activity_type]; + styles.color = data.properties.colors[feature.properties.activity_type]; return styles; } @@ -52,30 +52,35 @@ function loadData() { function onEachFeature(feature, layer) { layer.setStyle(styleTrack(feature)); layer.on('click', onClick); - if (feature.coordinates.length > 1) { - coords = coords.concat(feature.coordinates.map(function (p) { + if (feature.geometry.coordinates.length > 1) { + coords = coords.concat(feature.geometry.coordinates.map(function (p) { return [p[1], p[0], 0.1]; })); } } - var track = L.geoJSON(data['track'], { + var track = L.geoJSON(data['features'], { //style: styleTrack, onEachFeature: onEachFeature }).addTo(map); map.fitBounds(track.getBounds()); - var heat = L.heatLayer(coords); - L.control.layers(tiles, {"heatmap": heat}).addTo(map); + //var heat = L.heatLayer(coords); + //L.control.layers(tiles, {"heatmap": heat}).addTo(map); var list = $("