diff --git a/analyzers/__init__.py b/analyzers/__init__.py index 8bb53dd..c4b0ff6 100644 --- a/analyzers/__init__.py +++ b/analyzers/__init__.py @@ -2,14 +2,14 @@ from typing import List from .analyzer import Analyzer, Result from .analyzer.biogames import BoardDurationAnalyzer, SimulationRoundsAnalyzer, ActivationSequenceAnalyzer, \ - BiogamesCategorizer, ActivityMapper, BiogamesStore + BiogamesCategorizer, ActivityMapper, BiogamesStore, InstanceConfig, SimulationOrderAnalyzer from .analyzer.default import LogEntryCountAnalyzer, LocationAnalyzer, LogEntrySequenceAnalyzer, ActionSequenceAnalyzer, \ CategorizerStub, Store, ProgressAnalyzer from .analyzer.locomotion import LocomotionActionAnalyzer, CacheSequenceAnalyzer from .analyzer.mask import MaskSpatials from .render import Render from .render.biogames import SimulationRoundsRender, BoardDurationHistRender, BoardDurationBoxRender, \ - ActivityMapperRender, StoreRender, SimulationRoundsMeanRender + ActivityMapperRender, StoreRender, SimulationRoundsMeanRender, SimulationOrderRender from .render.default import PrintRender, JSONRender, TrackRender, HeatMapRender from .render.locomotion import LocomotionActionRelativeRender, LocomotionActionAbsoluteRender, \ LocomotionActionRatioRender, LocomotionActionRatioHistRender @@ -48,6 +48,10 @@ __MAPPING__ = { ], ProgressAnalyzer: [ StoreRender + ], + SimulationOrderAnalyzer: [ + JSONRender, +SimulationOrderRender ] } diff --git a/analyzers/analyzer/__init__.py b/analyzers/analyzer/__init__.py index 5ccea84..39050ba 100644 --- a/analyzers/analyzer/__init__.py +++ b/analyzers/analyzer/__init__.py @@ -43,7 +43,7 @@ class ResultStore: def get_store(self) -> dict: return dict(self.store) - def get_all(self) -> list: + def get_all(self) -> Collection[Result]: """ Throw all categories together :return: diff --git a/analyzers/analyzer/biogames.py b/analyzers/analyzer/biogames.py index 263bd62..96ad8f5 100644 --- a/analyzers/analyzer/biogames.py +++ b/analyzers/analyzer/biogames.py @@ -119,9 +119,7 @@ class ActivityMapper(Analyzer): def result(self, store: ResultStore) -> None: instance_config_id = self.instance_config_id for active_segment in self.store: # active_segment → sequence or None (None → map active) - host = self.settings.custom["host"] - seq_data_url = "{host}/game2/editor/config/{config_id}/sequence/{sequence_id}/".format( - host=host, + seq_data_url = "/game2/editor/config/{config_id}/sequence/{sequence_id}/".format( config_id=instance_config_id, sequence_id=active_segment.sequence, ) @@ -138,7 +136,7 @@ class ActivityMapper(Analyzer): if event[self.settings.type_field] in self.settings.boards: sequence_id = active_segment.sequence board_id = event["board_id"] - local_file = download_board(board_id, host, instance_config_id, sequence_id, source) + local_file = download_board(board_id, instance_config_id, sequence_id, source) if local_file is not None: event["image"] = local_file[16:] store.add(Result(type(self), {"instance": instance_config_id, "store": [x._asdict() for x in self.store]})) @@ -181,7 +179,6 @@ class BiogamesStore(Store): board_id = event["board_id"] local_file = download_board( board_id=board_id, - host=self.settings.custom["host"], instance_config_id=json_path(self.store[0], self.settings.custom["instance_config_id"]), sequence_id=sequence_id, source=self.settings.source) @@ -193,4 +190,42 @@ class BiogamesStore(Store): def process(self, entry: dict) -> bool: self.store.append(entry) + return False + + +class InstanceConfig(Analyzer): + __name__ = "InstanceConfig" + + def __init__(self, settings: LogSettings): + super().__init__(settings) + self.store = {} + + def process(self, entry: dict): + if entry[self.settings.type_field] in self.settings.custom["instance_start"]: + print(entry) + self.store["instance_id"] = json_path(entry, self.settings.custom["instance_config_id"]) + + def result(self, store: ResultStore): + store.add(Result(type(self), dict(self.store))) + + +class SimulationOrderAnalyzer(Analyzer): + __name__ = "SimuOrder" + + def __init__(self, settings: LogSettings): + super().__init__(settings) + self.store = defaultdict(lambda: -1) # TODO verify + self.order = [] + + def result(self, store: ResultStore) -> None: + store.add(Result(type(self), [self.store[sim] for sim in self.order])) + + def process(self, entry: dict) -> bool: + entry_type = entry[self.settings.type_field] + if entry_type in self.settings.custom['simulation_rounds']: + if entry["answers"][self.settings.type_field] in self.settings.custom["simu_data"]: + simu_id = entry['answers']["@id"] + self.store[simu_id] += 1 + if not simu_id in self.order: + self.order.append(simu_id) return False \ No newline at end of file diff --git a/analyzers/render/biogames.py b/analyzers/render/biogames.py index 48d27bd..e62bcfc 100644 --- a/analyzers/render/biogames.py +++ b/analyzers/render/biogames.py @@ -5,14 +5,15 @@ from typing import List, Tuple import matplotlib.pyplot as plt import os -from analyzers import Store, BiogamesStore +from analyzers import Store, BiogamesStore, SimulationOrderAnalyzer from . import Render from .. import Result, SimulationRoundsAnalyzer, BoardDurationAnalyzer, ActivityMapper - ONE_DAY = 24 * 60 * 60 -def plot(src_data: List[Tuple[str, List[int]]], title:str="simulation retries", ylabel:str="simulation rounds", xargs={}): + +def plot(src_data: List[Tuple[str, List[int]]], title: str = "simulation retries", ylabel: str = "simulation rounds", + xargs={}): names, datas = list(zip(*src_data)) plt.boxplot(datas, labels=names, **xargs) plt.xticks(rotation='vertical') @@ -22,9 +23,21 @@ def plot(src_data: List[Tuple[str, List[int]]], title:str="simulation retries", plt.show() +def plot_old(src_data: List[Tuple[str, List[int]]], ylabel="simulation rounds", title="simulation retries", + rotation='vertical'): + names, datas = list(zip(*src_data)) + plt.boxplot(datas, labels=names, **{"showfliers": True}) + plt.xticks(rotation=rotation) + # plt.margins() + plt.ylabel(ylabel) + plt.title(title) + plt.show() + + class SimulationRoundsRender(Render): result_types = [SimulationRoundsAnalyzer] xargs = {} + def render(self, results: List[Result]): data = defaultdict(list) for result in self.filter(results): @@ -37,6 +50,7 @@ class SimulationRoundsRender(Render): result_types = [SimulationRoundsAnalyzer] + class SimulationRoundsMeanRender(SimulationRoundsRender): xargs = {"showfliers": False} @@ -89,7 +103,7 @@ class ActivityMapperRender(Render): print(os.getcwd()) for result in self.filter(results): data = result.get() - with open(os.path.join("static", "progress", "data", data['instance']),"w") as out: + with open(os.path.join("static", "progress", "data", data['instance']), "w") as out: json.dump(data["store"], out, indent=1) return "ok" @@ -99,5 +113,20 @@ class StoreRender(Render): def render(self, results: List[Result]): for result in self.filter(results): - with open(os.path.join("static","progress","data","fooo"), "w") as out: + with open(os.path.join("static", "progress", "data", "fooo"), "w") as out: json.dump(result.get(), out, indent=1) + + +class SimulationOrderRender(Render): + def render(self, results: List[Result]): + data = defaultdict(list) + for result in self.filter(results): + get = result.get() + for i, value in enumerate(get): + data[i].append(value) + # data_tuples = [(key, data[key]) for key in sorted(data)] + # data_tuples = sorted(data_tuples, key=lambda x: sum(x[1])) + # plot(enumerate([r.get() for r in self.filter(results)])) + plot_old(list(data.items()), ylabel="simulation retries", title="sequential simulation retries", rotation=None) + + result_types = [SimulationOrderAnalyzer] diff --git a/biogames2.json b/biogames2.json index 3ee97c7..3a76e5b 100644 --- a/biogames2.json +++ b/biogames2.json @@ -14,15 +14,15 @@ "analyzers": { "analyzers": [ "BiogamesCategorizer", - "ActionSequenceAnalyzer", - "BoardDurationAnalyzer", - "LocomotionActionAnalyzer", - "CacheSequenceAnalyzer", - "SimulationRoundsAnalyzer", - "ActivationSequenceAnalyzer", - "ProgressAnalyzer" + "ActivityMapper", + "ProgressAnalyzer", + "SimulationOrderAnalyzer" ] }, + "dis":[ + "ActivityMapper", + "ProgressAnalyzer", + "InstanceConfig"], "disabled_analyzers": [ "ActivityMapper", "LocomotionActionAnalyzer", @@ -65,14 +65,13 @@ "action":"PAUSE" } }, - "host":"http://potato.kinf.wiai.uni-bamberg.de:5000", - "host2":"http://potato.kinf.wiai.uni-bamberg.de:5000", "coordinates": "location.coordinates" }, "source":{ "type": "Biogames", "url": "http://potato.kinf.wiai.uni-bamberg.de:5000/game2/instance/log/list/", "login_url": "http://potato.kinf.wiai.uni-bamberg.de:5000/game2/auth/json-login", + "host":"http://potato.kinf.wiai.uni-bamberg.de:5000", "username": "lb", "password": "81743" } diff --git a/log_analyzer.py b/log_analyzer.py index ff25e77..d9fe24a 100644 --- a/log_analyzer.py +++ b/log_analyzer.py @@ -130,9 +130,9 @@ if __name__ == '__main__': # "57c444470dbf88605433ca935c", # "78e0c545b594e82edfad55bd7f", # "91abfd4b31a5562b1c66be37d9", - # "597b704fe9ace475316c345903", - # "e01a684aa29dff9ddd9705edf8", - # "fbf9d64ae0bdad0de7efa3eec6", + "597b704fe9ace475316c345903", + "e01a684aa29dff9ddd9705edf8", + "fbf9d64ae0bdad0de7efa3eec6", # "fe1331481f85560681f86827ec", # "fe1331481f85560681f86827ec"] #"fec57041458e6cef98652df625", ] @@ -165,6 +165,13 @@ if __name__ == '__main__': print(str(a)) render(a, store.get_all()) + if False: + from analyzers.postprocessing import graph + g = graph.Cache(settings) + g.run(store) + if True: + render(analyzers.SimulationOrderAnalyzer, store.get_all()) + # for analyzers in analyzers: # if analyzers.name() in ["LogEntryCount", "ActionSequenceAnalyzer"]: # print(json.dumps(analyzers.result(), indent=2)) diff --git a/requirements.txt b/requirements.txt index 45e6fe7..dc88295 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,6 @@ -requests -numpy -matplotlib \ No newline at end of file +requests==2.18.4 +numpy==1.13.1 +matplotlib==2.1.0 +osmnx==0.6 +networkx==2.0 +pydot==1.2.3 \ No newline at end of file diff --git a/sources/biogames.py b/sources/biogames.py index d6c1798..e63a328 100644 --- a/sources/biogames.py +++ b/sources/biogames.py @@ -18,9 +18,10 @@ class Biogames(Source): self.headers: typing.Dict[str, str] = {'Accept': 'application/json'} self.cookies: typing.Dict[str, str] = {} self.id2link: typing.Dict[str, str] = {} + self.host: str = None def connect(self, **kwargs): - for i in ['username', 'password', 'url', 'login_url']: + for i in ['username', 'password', 'url', 'login_url', 'host']: if not i in kwargs: raise ValueError("missing value " + i) csrf_request = requests.get(kwargs['url']) @@ -40,12 +41,11 @@ class Biogames(Source): self.cookies['sessionid'] = login.cookies['sessionid'] log.info("obtained sessionid (" + self.cookies['sessionid'] + ")") self.url = kwargs['url'] + self.host = kwargs['host'] log.info("stored url (" + self.url + ")") def list(self): - logs_query = requests.get(self.url, cookies=self.cookies, headers=self.headers) - log.info(logs_query.status_code) - logs = logs_query.json() + logs = self.get_json(self.url) log.info(len(logs)) for i in logs: self.id2link[i["id"]] = i["link"] # TODO @@ -65,15 +65,18 @@ class Biogames(Source): def download_file(self, url, filename): with open(filename, "wb") as out: try: - download = requests.get(url, cookies=self.cookies, stream=True) + download = self._get(url) shutil.copyfileobj(download.raw, out) return filename except Exception as e: log.exception(e) os.remove(filename) + def get_json(self, url): + return self._get(url, stream=False).json() + def close(self): pass - def _get(self, url): - return requests.get(url, cookies=self.cookies, headers=self.headers, stream=True) + def _get(self, url, stream=True): + return requests.get(self.host + url, cookies=self.cookies, headers=self.headers, stream=stream) diff --git a/sources/source.py b/sources/source.py index 3827fa1..905b897 100644 --- a/sources/source.py +++ b/sources/source.py @@ -11,5 +11,8 @@ class Source: def get(self, ids: typing.Collection): raise NotImplementedError + def get_json(self, url:str) -> dict: + raise NotImplementedError + def close(self): raise NotImplementedError diff --git a/util/download.py b/util/download.py index daa9d48..703b658 100644 --- a/util/download.py +++ b/util/download.py @@ -6,7 +6,7 @@ from util import json_path logger = logging.getLogger(__name__) -def download_board(board_id, host, instance_config_id, sequence_id, source): +def download_board(board_id, instance_config_id, sequence_id, source): if sequence_id is None: return "static" local_file = "static/progress/images/{config_id}/{sequence_id}/{board_id}".format( @@ -15,8 +15,7 @@ def download_board(board_id, host, instance_config_id, sequence_id, source): board_id=board_id) if os.path.exists(local_file): return local_file - url = "{host}/game2/editor/config/{config_id}/sequence/{sequence_id}/board/{board_id}/".format( - host=host, + url = "/game2/editor/config/{config_id}/sequence/{sequence_id}/board/{board_id}/".format( config_id=instance_config_id, sequence_id=sequence_id, board_id=board_id @@ -30,5 +29,18 @@ def download_board(board_id, host, instance_config_id, sequence_id, source): preview_url = json_path(data, "preview_url.medium") logger.debug(preview_url) os.makedirs(local_file[:-len(board_id)], exist_ok=True) - source.download_file(host + preview_url, local_file) + source.download_file(preview_url, local_file) return local_file + + +def get_config(source, instance_id): + url = "/game2/editor/config/{config_id}/".format(config_id=instance_id) + instance_data = source.get_json(url) + caches = url + "cache/" + cache_data = source.get_json(caches) + + return { + "name": instance_data["name"], + "id": instance_data["@id"], + "caches": cache_data + }