diff --git a/analyzers/__init__.py b/analyzers/__init__.py index 1a098ff..5a95e21 100644 --- a/analyzers/__init__.py +++ b/analyzers/__init__.py @@ -2,14 +2,14 @@ from typing import List from .analyzer import Analyzer, Result from .analyzer.biogames import BoardDurationAnalyzer, SimulationRoundsAnalyzer, ActivationSequenceAnalyzer, \ - BiogamesCategorizer, ActivityMapper, BiogamesStore + BiogamesCategorizer, ActivityMapper, BiogamesStore, InstanceConfig, SimulationOrderAnalyzer from .analyzer.default import LogEntryCountAnalyzer, LocationAnalyzer, LogEntrySequenceAnalyzer, ActionSequenceAnalyzer, \ CategorizerStub, Store, ProgressAnalyzer from .analyzer.locomotion import LocomotionActionAnalyzer, CacheSequenceAnalyzer from .analyzer.mask import MaskSpatials from .render import Render from .render.biogames import SimulationRoundsRender, BoardDurationHistRender, BoardDurationBoxRender, \ - ActivityMapperRender, StoreRender + ActivityMapperRender, StoreRender, SimulationOrderRender from .render.default import PrintRender, JSONRender, TrackRender, HeatMapRender from .render.locomotion import LocomotionActionRelativeRender, LocomotionActionAbsoluteRender, \ LocomotionActionRatioRender @@ -46,6 +46,10 @@ __MAPPING__ = { ], ProgressAnalyzer: [ StoreRender + ], + SimulationOrderAnalyzer: [ + JSONRender, +SimulationOrderRender ] } diff --git a/analyzers/analyzer/__init__.py b/analyzers/analyzer/__init__.py index bdfe6db..3b8e54f 100644 --- a/analyzers/analyzer/__init__.py +++ b/analyzers/analyzer/__init__.py @@ -44,7 +44,7 @@ class ResultStore: def get_store(self) -> dict: return dict(self.store) - def get_all(self) -> list: + def get_all(self) -> Collection[Result]: """ Throw all categories together :return: diff --git a/analyzers/analyzer/biogames.py b/analyzers/analyzer/biogames.py index 57fb2cb..79b21a9 100644 --- a/analyzers/analyzer/biogames.py +++ b/analyzers/analyzer/biogames.py @@ -119,9 +119,7 @@ class ActivityMapper(Analyzer): def result(self, store: ResultStore) -> None: instance_config_id = self.instance_config_id for active_segment in self.store: # active_segment → sequence or None (None → map active) - host = self.settings.custom["host"] - seq_data_url = "{host}/game2/editor/config/{config_id}/sequence/{sequence_id}/".format( - host=host, + seq_data_url = "/game2/editor/config/{config_id}/sequence/{sequence_id}/".format( config_id=instance_config_id, sequence_id=active_segment.sequence, ) @@ -133,7 +131,7 @@ class ActivityMapper(Analyzer): if event[self.settings.type_field] in self.settings.boards: sequence_id = active_segment.sequence board_id = event["board_id"] - local_file = download_board(board_id, host, instance_config_id, sequence_id, source) + local_file = download_board(board_id, instance_config_id, sequence_id, source) if local_file is not None: event["image"] = local_file[16:] store.add(Result(type(self), {"instance": instance_config_id, "store": [x._asdict() for x in self.store]})) @@ -176,7 +174,6 @@ class BiogamesStore(Store): board_id = event["board_id"] local_file = download_board( board_id=board_id, - host=self.settings.custom["host"], instance_config_id=json_path(self.store[0], self.settings.custom["instance_config_id"]), sequence_id=sequence_id, source=self.settings.source) @@ -188,4 +185,42 @@ class BiogamesStore(Store): def process(self, entry: dict) -> bool: self.store.append(entry) + return False + + +class InstanceConfig(Analyzer): + __name__ = "InstanceConfig" + + def __init__(self, settings: LogSettings): + super().__init__(settings) + self.store = {} + + def process(self, entry: dict): + if entry[self.settings.type_field] in self.settings.custom["instance_start"]: + print(entry) + self.store["instance_id"] = json_path(entry, self.settings.custom["instance_config_id"]) + + def result(self, store: ResultStore): + store.add(Result(type(self), dict(self.store))) + + +class SimulationOrderAnalyzer(Analyzer): + __name__ = "SimuOrder" + + def __init__(self, settings: LogSettings): + super().__init__(settings) + self.store = defaultdict(lambda: -1) # TODO verify + self.order = [] + + def result(self, store: ResultStore) -> None: + store.add(Result(type(self), [self.store[sim] for sim in self.order])) + + def process(self, entry: dict) -> bool: + entry_type = entry[self.settings.type_field] + if entry_type in self.settings.custom['simulation_rounds']: + if entry["answers"][self.settings.type_field] in self.settings.custom["simu_data"]: + simu_id = entry['answers']["@id"] + self.store[simu_id] += 1 + if not simu_id in self.order: + self.order.append(simu_id) return False \ No newline at end of file diff --git a/analyzers/render/biogames.py b/analyzers/render/biogames.py index 7aea6ab..65e8fe7 100644 --- a/analyzers/render/biogames.py +++ b/analyzers/render/biogames.py @@ -5,18 +5,19 @@ from typing import List, Tuple import matplotlib.pyplot as plt import os -from analyzers import Store, BiogamesStore +from analyzers import Store, BiogamesStore, SimulationOrderAnalyzer from . import Render from .. import Result, SimulationRoundsAnalyzer, BoardDurationAnalyzer, ActivityMapper -def plot(src_data: List[Tuple[str, List[int]]]): +def plot(src_data: List[Tuple[str, List[int]]], ylabel="simulation rounds", title="simulation retries", + rotation='vertical'): names, datas = list(zip(*src_data)) plt.boxplot(datas, labels=names) - plt.xticks(rotation='vertical') + plt.xticks(rotation=rotation) # plt.margins() - plt.ylabel("simulation rounds") - plt.title("simulation retries") + plt.ylabel(ylabel) + plt.title(title) plt.show() @@ -86,3 +87,18 @@ class StoreRender(Render): for result in self.filter(results): with open(os.path.join("static","progress","data","fooo"), "w") as out: json.dump(result.get(), out, indent=1) + + +class SimulationOrderRender(Render): + def render(self, results: List[Result]): + data = defaultdict(list) + for result in self.filter(results): + get = result.get() + for i,value in enumerate(get): + data[i].append(value) + #data_tuples = [(key, data[key]) for key in sorted(data)] + #data_tuples = sorted(data_tuples, key=lambda x: sum(x[1])) + #plot(enumerate([r.get() for r in self.filter(results)])) + plot(list(data.items()), ylabel="simulation retries", title="sequential simulation retries", rotation=None) + + result_types = [SimulationOrderAnalyzer] \ No newline at end of file diff --git a/biogames2.json b/biogames2.json index 1bebe72..2dfb70f 100644 --- a/biogames2.json +++ b/biogames2.json @@ -14,10 +14,13 @@ "analyzers": { "analyzers": [ "BiogamesCategorizer", - "ActivityMapper", - "ProgressAnalyzer" + "SimulationOrderAnalyzer" ] }, + "dis":[ + "ActivityMapper", + "ProgressAnalyzer", + "InstanceConfig"], "disabled_analyzers": [ "LocomotionActionAnalyzer", "LogEntryCountAnalyzer", @@ -59,7 +62,6 @@ "action":"PAUSE" } }, - "host":"http://0.0.0.0:5000", "coordinates": "location.coordinates" }, "source":{ @@ -67,6 +69,7 @@ "url": "http://0.0.0.0:5000/game2/instance/log/list/", "login_url": "http://localhost:5000/game2/auth/json-login", "username": "dev", - "password": "dev" + "password": "dev", + "host":"http://0.0.0.0:5000" } } \ No newline at end of file diff --git a/log_analyzer.py b/log_analyzer.py index d650e7f..cc0f71c 100644 --- a/log_analyzer.py +++ b/log_analyzer.py @@ -48,9 +48,9 @@ if __name__ == '__main__': # "57c444470dbf88605433ca935c", # "78e0c545b594e82edfad55bd7f", # "91abfd4b31a5562b1c66be37d9", - # "597b704fe9ace475316c345903", - # "e01a684aa29dff9ddd9705edf8", - # "fbf9d64ae0bdad0de7efa3eec6", + "597b704fe9ace475316c345903", + "e01a684aa29dff9ddd9705edf8", + "fbf9d64ae0bdad0de7efa3eec6", # "fe1331481f85560681f86827ec", "fe1331481f85560681f86827ec"] #"fec57041458e6cef98652df625", ] @@ -67,10 +67,17 @@ if __name__ == '__main__': if False: render(analyzers.LocationAnalyzer, store.get_all()) #print(json.dumps(store.serializable(), indent=1)) - if True: + if False: render(analyzers.ActivityMapper, store.get_all()) render(analyzers.ProgressAnalyzer, store.get_all()) + if False: + from analyzers.postprocessing import graph + g = graph.Cache(settings) + g.run(store) + if True: + render(analyzers.SimulationOrderAnalyzer, store.get_all()) + # for analyzers in analyzers: # if analyzers.name() in ["LogEntryCount", "ActionSequenceAnalyzer"]: # print(json.dumps(analyzers.result(), indent=2)) diff --git a/requirements.txt b/requirements.txt index 45e6fe7..dc88295 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,6 @@ -requests -numpy -matplotlib \ No newline at end of file +requests==2.18.4 +numpy==1.13.1 +matplotlib==2.1.0 +osmnx==0.6 +networkx==2.0 +pydot==1.2.3 \ No newline at end of file diff --git a/sources/biogames.py b/sources/biogames.py index d6c1798..e63a328 100644 --- a/sources/biogames.py +++ b/sources/biogames.py @@ -18,9 +18,10 @@ class Biogames(Source): self.headers: typing.Dict[str, str] = {'Accept': 'application/json'} self.cookies: typing.Dict[str, str] = {} self.id2link: typing.Dict[str, str] = {} + self.host: str = None def connect(self, **kwargs): - for i in ['username', 'password', 'url', 'login_url']: + for i in ['username', 'password', 'url', 'login_url', 'host']: if not i in kwargs: raise ValueError("missing value " + i) csrf_request = requests.get(kwargs['url']) @@ -40,12 +41,11 @@ class Biogames(Source): self.cookies['sessionid'] = login.cookies['sessionid'] log.info("obtained sessionid (" + self.cookies['sessionid'] + ")") self.url = kwargs['url'] + self.host = kwargs['host'] log.info("stored url (" + self.url + ")") def list(self): - logs_query = requests.get(self.url, cookies=self.cookies, headers=self.headers) - log.info(logs_query.status_code) - logs = logs_query.json() + logs = self.get_json(self.url) log.info(len(logs)) for i in logs: self.id2link[i["id"]] = i["link"] # TODO @@ -65,15 +65,18 @@ class Biogames(Source): def download_file(self, url, filename): with open(filename, "wb") as out: try: - download = requests.get(url, cookies=self.cookies, stream=True) + download = self._get(url) shutil.copyfileobj(download.raw, out) return filename except Exception as e: log.exception(e) os.remove(filename) + def get_json(self, url): + return self._get(url, stream=False).json() + def close(self): pass - def _get(self, url): - return requests.get(url, cookies=self.cookies, headers=self.headers, stream=True) + def _get(self, url, stream=True): + return requests.get(self.host + url, cookies=self.cookies, headers=self.headers, stream=stream) diff --git a/sources/source.py b/sources/source.py index 3827fa1..905b897 100644 --- a/sources/source.py +++ b/sources/source.py @@ -11,5 +11,8 @@ class Source: def get(self, ids: typing.Collection): raise NotImplementedError + def get_json(self, url:str) -> dict: + raise NotImplementedError + def close(self): raise NotImplementedError diff --git a/util/download.py b/util/download.py index f0f2998..78dadc5 100644 --- a/util/download.py +++ b/util/download.py @@ -6,15 +6,14 @@ from util import json_path logger = logging.getLogger(__name__) -def download_board(board_id, host, instance_config_id, sequence_id, source): +def download_board(board_id, instance_config_id, sequence_id, source): local_file = "static/progress/images/{config_id}/{sequence_id}/{board_id}".format( config_id=instance_config_id, sequence_id=sequence_id, board_id=board_id) if os.path.exists(local_file): return local_file - url = "{host}/game2/editor/config/{config_id}/sequence/{sequence_id}/board/{board_id}/".format( - host=host, + url = "/game2/editor/config/{config_id}/sequence/{sequence_id}/board/{board_id}/".format( config_id=instance_config_id, sequence_id=sequence_id, board_id=board_id @@ -26,5 +25,18 @@ def download_board(board_id, host, instance_config_id, sequence_id, source): preview_url = json_path(data, "preview_url.medium") logger.debug(preview_url) os.makedirs(local_file[:-len(board_id)], exist_ok=True) - source.download_file(host + preview_url, local_file) + source.download_file(preview_url, local_file) return local_file + + +def get_config(source, instance_id): + url = "/game2/editor/config/{config_id}/".format(config_id=instance_id) + instance_data = source.get_json(url) + caches = url + "cache/" + cache_data = source.get_json(caches) + + return { + "name": instance_data["name"], + "id": instance_data["@id"], + "caches": cache_data + }