add SequenceSimuRetries

analyzers/__init__.py

@@ -2,14 +2,14 @@ from typing import List
 
 from .analyzer import Analyzer, Result
 from .analyzer.biogames import BoardDurationAnalyzer, SimulationRoundsAnalyzer, ActivationSequenceAnalyzer, \
-	BiogamesCategorizer, ActivityMapper, BiogamesStore
+	BiogamesCategorizer, ActivityMapper, BiogamesStore, InstanceConfig, SimulationOrderAnalyzer
 from .analyzer.default import LogEntryCountAnalyzer, LocationAnalyzer, LogEntrySequenceAnalyzer, ActionSequenceAnalyzer, \
 	CategorizerStub, Store, ProgressAnalyzer
 from .analyzer.locomotion import LocomotionActionAnalyzer, CacheSequenceAnalyzer
 from .analyzer.mask import MaskSpatials
 from .render import Render
 from .render.biogames import SimulationRoundsRender, BoardDurationHistRender, BoardDurationBoxRender, \
-	ActivityMapperRender, StoreRender
+	ActivityMapperRender, StoreRender, SimulationOrderRender
 from .render.default import PrintRender, JSONRender, TrackRender, HeatMapRender
 from .render.locomotion import LocomotionActionRelativeRender, LocomotionActionAbsoluteRender, \
 	LocomotionActionRatioRender
@@ -46,6 +46,10 @@ __MAPPING__ = {
 	],
 	ProgressAnalyzer: [
 		StoreRender
+	],
+	SimulationOrderAnalyzer: [
+		JSONRender,
+SimulationOrderRender
 	]
 }
 

analyzers/analyzer/__init__.py

@@ -44,7 +44,7 @@ class ResultStore:
 	def get_store(self) -> dict:
 		return dict(self.store)
 
-	def get_all(self) -> list:
+	def get_all(self) -> Collection[Result]:
 		"""
 		Throw all categories together
 		:return:

analyzers/analyzer/biogames.py

@@ -119,9 +119,7 @@ class ActivityMapper(Analyzer):
 	def result(self, store: ResultStore) -> None:
 		instance_config_id = self.instance_config_id
 		for active_segment in self.store:  # active_segment → sequence or None (None → map active)
-			host = self.settings.custom["host"]
+			seq_data_url = "/game2/editor/config/{config_id}/sequence/{sequence_id}/".format(
-			seq_data_url = "{host}/game2/editor/config/{config_id}/sequence/{sequence_id}/".format(
-				host=host,
 				config_id=instance_config_id,
 				sequence_id=active_segment.sequence,
 			)
@@ -133,7 +131,7 @@ class ActivityMapper(Analyzer):
 				if event[self.settings.type_field] in self.settings.boards:
 					sequence_id = active_segment.sequence
 					board_id = event["board_id"]
-					local_file = download_board(board_id, host, instance_config_id, sequence_id, source)
+					local_file = download_board(board_id, instance_config_id, sequence_id, source)
 					if local_file is not None:
 						event["image"] = local_file[16:]
 		store.add(Result(type(self), {"instance": instance_config_id, "store": [x._asdict() for x in self.store]}))
@@ -176,7 +174,6 @@ class BiogamesStore(Store):
 				board_id = event["board_id"]
 				local_file = download_board(
 					board_id=board_id,
-					host=self.settings.custom["host"],
 					instance_config_id=json_path(self.store[0], self.settings.custom["instance_config_id"]),
 					sequence_id=sequence_id,
 					source=self.settings.source)
@@ -188,4 +185,42 @@ class BiogamesStore(Store):
 
 	def process(self, entry: dict) -> bool:
 		self.store.append(entry)
+		return False
+
+
+class InstanceConfig(Analyzer):
+	__name__ = "InstanceConfig"
+
+	def __init__(self, settings: LogSettings):
+		super().__init__(settings)
+		self.store = {}
+
+	def process(self, entry: dict):
+		if entry[self.settings.type_field] in self.settings.custom["instance_start"]:
+			print(entry)
+			self.store["instance_id"] = json_path(entry, self.settings.custom["instance_config_id"])
+
+	def result(self, store: ResultStore):
+		store.add(Result(type(self), dict(self.store)))
+
+
+class SimulationOrderAnalyzer(Analyzer):
+	__name__ = "SimuOrder"
+
+	def __init__(self, settings: LogSettings):
+		super().__init__(settings)
+		self.store = defaultdict(lambda: -1)  # TODO verify
+		self.order = []
+
+	def result(self, store: ResultStore) -> None:
+		store.add(Result(type(self), [self.store[sim] for sim in self.order]))
+
+	def process(self, entry: dict) -> bool:
+		entry_type = entry[self.settings.type_field]
+		if entry_type in self.settings.custom['simulation_rounds']:
+			if entry["answers"][self.settings.type_field] in self.settings.custom["simu_data"]:
+				simu_id = entry['answers']["@id"]
+				self.store[simu_id] += 1
+				if not simu_id in self.order:
+					self.order.append(simu_id)
 		return False

analyzers/render/biogames.py

@@ -5,18 +5,19 @@ from typing import List, Tuple
 import matplotlib.pyplot as plt
 import os
 
-from analyzers import Store, BiogamesStore
+from analyzers import Store, BiogamesStore, SimulationOrderAnalyzer
 from . import Render
 from .. import Result, SimulationRoundsAnalyzer, BoardDurationAnalyzer, ActivityMapper
 
 
-def plot(src_data: List[Tuple[str, List[int]]]):
+def plot(src_data: List[Tuple[str, List[int]]], ylabel="simulation rounds", title="simulation retries",
+         rotation='vertical'):
 	names, datas = list(zip(*src_data))
 	plt.boxplot(datas, labels=names)
-	plt.xticks(rotation='vertical')
+	plt.xticks(rotation=rotation)
 	# plt.margins()
-	plt.ylabel("simulation rounds")
+	plt.ylabel(ylabel)
-	plt.title("simulation retries")
+	plt.title(title)
 	plt.show()
 
 
@@ -86,3 +87,18 @@ class StoreRender(Render):
 		for result in self.filter(results):
 			with open(os.path.join("static","progress","data","fooo"), "w") as out:
 				json.dump(result.get(), out, indent=1)
+
+
+class SimulationOrderRender(Render):
+	def render(self, results: List[Result]):
+		data = defaultdict(list)
+		for result in self.filter(results):
+			get = result.get()
+			for i,value in enumerate(get):
+				data[i].append(value)
+		#data_tuples = [(key, data[key]) for key in sorted(data)]
+		#data_tuples = sorted(data_tuples, key=lambda x: sum(x[1]))
+		#plot(enumerate([r.get() for r in self.filter(results)]))
+		plot(list(data.items()), ylabel="simulation retries", title="sequential simulation retries", rotation=None)
+
+	result_types = [SimulationOrderAnalyzer]

biogames2.json

@@ -14,10 +14,13 @@
   "analyzers": {
     "analyzers": [
       "BiogamesCategorizer",
-      "ActivityMapper",
+      "SimulationOrderAnalyzer"
-      "ProgressAnalyzer"
     ]
   },
+  "dis":[
+      "ActivityMapper",
+      "ProgressAnalyzer",
+      "InstanceConfig"],
   "disabled_analyzers": [
       "LocomotionActionAnalyzer",
       "LogEntryCountAnalyzer",
@@ -59,7 +62,6 @@
         "action":"PAUSE"
       }
     },
-    "host":"http://0.0.0.0:5000",
     "coordinates": "location.coordinates"
   },
   "source":{
@@ -67,6 +69,7 @@
     "url": "http://0.0.0.0:5000/game2/instance/log/list/",
     "login_url": "http://localhost:5000/game2/auth/json-login",
     "username": "dev",
-    "password": "dev"
+    "password": "dev",
+    "host":"http://0.0.0.0:5000"
   }
 }

log_analyzer.py

@@ -48,9 +48,9 @@ if __name__ == '__main__':
 		#	"57c444470dbf88605433ca935c",
 		#	"78e0c545b594e82edfad55bd7f",
 		#	"91abfd4b31a5562b1c66be37d9",
-		#	"597b704fe9ace475316c345903",
+			"597b704fe9ace475316c345903",
-		#	"e01a684aa29dff9ddd9705edf8",
+			"e01a684aa29dff9ddd9705edf8",
-		#	"fbf9d64ae0bdad0de7efa3eec6",
+			"fbf9d64ae0bdad0de7efa3eec6",
 		#	"fe1331481f85560681f86827ec",
 			"fe1331481f85560681f86827ec"]
 		#"fec57041458e6cef98652df625", ]
@@ -67,10 +67,17 @@ if __name__ == '__main__':
 	if False:
 		render(analyzers.LocationAnalyzer, store.get_all())
 	#print(json.dumps(store.serializable(), indent=1))
-	if True:
+	if False:
 		render(analyzers.ActivityMapper, store.get_all())
 		render(analyzers.ProgressAnalyzer, store.get_all())
 
+	if False:
+		from analyzers.postprocessing import graph
+		g = graph.Cache(settings)
+		g.run(store)
+	if True:
+		render(analyzers.SimulationOrderAnalyzer, store.get_all())
+
 # for analyzers in analyzers:
 #	if analyzers.name() in ["LogEntryCount", "ActionSequenceAnalyzer"]:
 #		print(json.dumps(analyzers.result(), indent=2))

requirements.txt

@@ -1,3 +1,6 @@
-requests
+requests==2.18.4
-numpy
+numpy==1.13.1
-matplotlib
+matplotlib==2.1.0
+osmnx==0.6
+networkx==2.0
+pydot==1.2.3

sources/biogames.py

@@ -18,9 +18,10 @@ class Biogames(Source):
 		self.headers: typing.Dict[str, str] = {'Accept': 'application/json'}
 		self.cookies: typing.Dict[str, str] = {}
 		self.id2link: typing.Dict[str, str] = {}
+		self.host: str = None
 
 	def connect(self, **kwargs):
-		for i in ['username', 'password', 'url', 'login_url']:
+		for i in ['username', 'password', 'url', 'login_url', 'host']:
 			if not i in kwargs:
 				raise ValueError("missing value " + i)
 		csrf_request = requests.get(kwargs['url'])
@@ -40,12 +41,11 @@ class Biogames(Source):
 		self.cookies['sessionid'] = login.cookies['sessionid']
 		log.info("obtained sessionid (" + self.cookies['sessionid'] + ")")
 		self.url = kwargs['url']
+		self.host = kwargs['host']
 		log.info("stored url (" + self.url + ")")
 
 	def list(self):
-		logs_query = requests.get(self.url, cookies=self.cookies, headers=self.headers)
+		logs = self.get_json(self.url)
-		log.info(logs_query.status_code)
-		logs = logs_query.json()
 		log.info(len(logs))
 		for i in logs:
 			self.id2link[i["id"]] = i["link"]  # TODO
@@ -65,15 +65,18 @@ class Biogames(Source):
 	def download_file(self, url, filename):
 		with open(filename, "wb") as out:
 			try:
-				download = requests.get(url, cookies=self.cookies, stream=True)
+				download = self._get(url)
 				shutil.copyfileobj(download.raw, out)
 				return filename
 			except Exception as e:
 				log.exception(e)
 				os.remove(filename)
 
+	def get_json(self, url):
+		return self._get(url, stream=False).json()
+
 	def close(self):
 		pass
 
-	def _get(self, url):
+	def _get(self, url, stream=True):
-		return requests.get(url, cookies=self.cookies, headers=self.headers, stream=True)
+		return requests.get(self.host + url, cookies=self.cookies, headers=self.headers, stream=stream)

sources/source.py

@@ -11,5 +11,8 @@ class Source:
 	def get(self, ids: typing.Collection):
 		raise NotImplementedError
 
+	def get_json(self, url:str) -> dict:
+		raise NotImplementedError
+
 	def close(self):
 		raise NotImplementedError

util/download.py

@@ -6,15 +6,14 @@ from util import json_path
 logger = logging.getLogger(__name__)
 
 
-def download_board(board_id, host, instance_config_id, sequence_id, source):
+def download_board(board_id, instance_config_id, sequence_id, source):
 	local_file = "static/progress/images/{config_id}/{sequence_id}/{board_id}".format(
 		config_id=instance_config_id,
 		sequence_id=sequence_id,
 		board_id=board_id)
 	if os.path.exists(local_file):
 		return local_file
-	url = "{host}/game2/editor/config/{config_id}/sequence/{sequence_id}/board/{board_id}/".format(
+	url = "/game2/editor/config/{config_id}/sequence/{sequence_id}/board/{board_id}/".format(
-		host=host,
 		config_id=instance_config_id,
 		sequence_id=sequence_id,
 		board_id=board_id
@@ -26,5 +25,18 @@ def download_board(board_id, host, instance_config_id, sequence_id, source):
 	preview_url = json_path(data, "preview_url.medium")
 	logger.debug(preview_url)
 	os.makedirs(local_file[:-len(board_id)], exist_ok=True)
-	source.download_file(host + preview_url, local_file)
+	source.download_file(preview_url, local_file)
 	return local_file
+
+
+def get_config(source, instance_id):
+	url = "/game2/editor/config/{config_id}/".format(config_id=instance_id)
+	instance_data = source.get_json(url)
+	caches = url + "cache/"
+	cache_data = source.get_json(caches)
+
+	return {
+		"name": instance_data["name"],
+		"id": instance_data["@id"],
+		"caches": cache_data
+	}