add code to understand/explore simulationData flags

analyzers/__init__.py

@@ -2,7 +2,8 @@ from typing import List
 
 from .analyzer import Analyzer, Result
 from .analyzer.biogames import BoardDurationAnalyzer, SimulationRoundsAnalyzer, ActivationSequenceAnalyzer, \
-	BiogamesCategorizer, ActivityMapper, BiogamesStore, InstanceConfig, SimulationOrderAnalyzer, SimulationCategorizer
+	BiogamesCategorizer, ActivityMapper, BiogamesStore, InstanceConfig, SimulationOrderAnalyzer, SimulationCategorizer, \
+	SimulationFlagsAnalyzer
 from .analyzer.default import LogEntryCountAnalyzer, LocationAnalyzer, LogEntrySequenceAnalyzer, ActionSequenceAnalyzer, \
 	CategorizerStub, Store, ProgressAnalyzer
 from .analyzer.locomotion import LocomotionActionAnalyzer, CacheSequenceAnalyzer

analyzers/analyzer/__init__.py

@@ -8,10 +8,11 @@ log: logging.Logger = logging.getLogger(__name__)
 
 
 class Result:
-	def __init__(self, analysis: Type, result: Sized):
+	def __init__(self, analysis: Type, result: Sized, name: str = None):
 		self.result = result
 		self.__analysis__ = analysis
 		log.debug("set" + str(len(self.result)))
+		self.name = name
 
 	def analysis(self):
 		return self.__analysis__
@@ -21,7 +22,8 @@ class Result:
 		return self.result
 
 	def __repr__(self):
-		return "<Result " + str(self.__analysis__) + ": " + str(type(self.result)) + " " + str(len(self.result)) + ">"
+		return "<Result " + str(self.__analysis__) + ": " + str(type(self.result)) + " " + str(
+			len(self.result)) + " for " + str(self.name) + ">"
 
 
 class ResultStore:
@@ -66,7 +68,11 @@ class ResultStore:
 	def serializable(self):
 		values = {}
 		for key in self.store:
-			values[key] = [{"analysis": str(result.analysis()), "result": result.get()} for result in self.store[key]]
+			values[key] = [{
+				"analysis": str(result.analysis()),
+				"result": result.get(),
+				"name": result.name
+			} for result in self.store[key]]
 		return values
 
 
@@ -84,7 +90,7 @@ class Analyzer:
 		"""
 		raise NotImplementedError()
 
-	def result(self, store: ResultStore) -> None:
+	def result(self, store: ResultStore, name=None) -> None:
 		raise NotImplementedError()
 
 	def name(self) -> str:

analyzers/analyzer/biogames.py

@@ -305,10 +305,7 @@ class SimulationOrderAnalyzer(Analyzer):
 
 
 class SimulationCategorizer(CategorizerStub):  # TODO: refactor categorizer
-	__name__ = "SimulationCategorizer"
+	__name__ = "SimulationCategorizer"# TODO: rename -.- (InstanceConfigIDCategorizer)
-
-	def __init__(self, settings: LogSettings):
-		super().__init__(settings)
 
 	def process(self, entry: dict) -> bool:
 		if self.key is "default":
@@ -319,3 +316,23 @@ class SimulationCategorizer(CategorizerStub):  # TODO: refactor categorizer
 					print(entry)
 					raise e
 		return False
+
+
+class SimulationFlagsAnalyzer(Analyzer):
+	__name__ = "SimuFlags"
+
+	def __init__(self, settings: LogSettings) -> None:
+		super().__init__(settings)
+		self.store = []
+
+	def process(self, entry: dict) -> bool:
+		entry_type = entry[self.settings.type_field]
+		if entry_type in self.settings.custom['simulation_rounds']:
+			if entry["answers"][self.settings.type_field] in self.settings.custom["simu_data"]:
+				self.store.append(entry)
+		return False
+
+	def result(self, store: ResultStore, name=None) -> None:
+		store.add(Result(type(self), self.store, name=name))
+
+

analyzers/analyzer/default.py

@@ -89,7 +89,7 @@ class CategorizerStub(Analyzer):
 
 	__name__ = "Categorizer"
 
-	def result(self, store: ResultStore) -> None:
+	def result(self, store: ResultStore, name=None) -> None:
 		store.new_category(self.key)
 
 	def __init__(self, settings: LogSettings):

biogames2.json

@@ -14,10 +14,11 @@
   "analyzers": {
     "analyzers": [
       "SimulationCategorizer",
-      "ActivityMapper"
+      "SimulationFlagsAnalyzer"
     ]
   },
   "dis":[
+      "ActivityMapper",
       "BiogamesCategorizer",
       "LogEntryCountAnalyzer",
       "SimulationOrderAnalyzer",
@@ -70,8 +71,8 @@
     "type": "Biogames",
     "url": "http://0.0.0.0:5000/game2/instance/log/list/",
     "login_url": "http://localhost:5000/game2/auth/json-login",
-    "username": "ba",
+    "username": "dev",
-    "password": "853451",
+    "password": "dev",
     "host":"http://0.0.0.0:5000"
   }
 }

filter_todo/pre_filter.py

@@ -0,0 +1,90 @@
+import os
+from zipfile import ZipFile
+import sqlite3
+import json
+from collections import defaultdict
+
+
+def get_json(filename):
+	log = []
+	id = None
+	with ZipFile(filename) as zipf:
+		zipf.extract('instance_log.sqlite')
+		sql = sqlite3.connect('instance_log.sqlite')
+		cursor = sql.cursor()
+		for r in cursor.execute('SELECT json FROM log_entry;'):
+			entry = json.loads(r[0])
+			log.append(entry)
+			if id is None:
+				id = entry['instance_id']
+		sql.close()
+		os.remove('instance_log.sqlite')
+	return id, log
+
+
+def is_finished(log):
+	for entry in log:
+		if "action" in entry:
+			if "LogEntryInstanceAction" in entry["@class"] and entry["action"][
+				"@class"] == "de.findevielfalt.games.game2.instance.action.EndGameEnableAction" and entry['action'][
+				'enable']:
+				return True
+	return False
+
+
+def get_simus(log):
+	simus = defaultdict(lambda: 0)
+	order = []
+	actions = 0
+	for entry in log:
+		if "LogEntryQuestion" in entry["@class"]:
+			if "SimulationBoardData" in entry["answers"]["@class"]:
+				id = entry["answers"]["@id"]
+				simus[id] += 1
+				actions += 1 if entry['selected_actions'] else 0
+				if not id in order:
+					order.append(id)
+	return dict(simus), order, actions
+
+
+def simu_dist(simus):
+	dist = defaultdict(lambda: 0)
+	for instance in simus:
+		sim = simus[instance]
+		dist[len(sim)] += 1
+	return dist
+
+
+logs = {}
+finished = []
+simus = {}
+distribution = defaultdict(lambda: 0)
+finished_and_simu = defaultdict(list)
+files = {}
+actions_dist = defaultdict(list)
+with open('/home/agp8x/git/uni/ma/project/data/0000_ref') as src:
+	for line in src:
+		line = line.strip()
+		instance_id, log = get_json(line)
+		logs[instance_id] = log
+		files[instance_id] = line
+for id in logs:
+	simus[id] = get_simus(logs[id])
+	simu_count = len(simus[id][1])
+	distribution[simu_count] += 1
+	actions_dist[simus[id][2]].append(id)
+	if is_finished(logs[id]):
+		finished.append(id)
+		finished_and_simu[simu_count].append(id)
+print("total: ", len(logs))
+print("finished: ", len(finished))
+print("simu_dist: ", len(distribution), json.dumps(distribution, sort_keys=True))
+for i in sorted(finished_and_simu):
+	print("fin+sim" + str(i) + ": ", len(finished_and_simu[i]))
+for i in sorted(actions_dist):
+	print("actions: ", i, len(actions_dist[i]))
+print(json.dumps(actions_dist[4], sort_keys=True, indent=2))
+
+# print(finished_and_simu)
+# for instance in finished_and_simu:
+#	print(files[instance])

log_analyzer.py

@@ -63,7 +63,7 @@ if __name__ == '__main__':
 	# "fec57041458e6cef98652df625", ]
 	log_ids = []
 	# with open("/home/clemens/git/ma/test/filtered") as src:
-	with open("/home/clemens/git/ma/test/filtered") as src:
+	with open("/home/agp8x/git/uni/ma/project/data/0000_ref") as src:
 		for line in src:
 			line = line.strip()
 			log_ids.append(line)
@@ -73,7 +73,7 @@ if __name__ == '__main__':
 			log.info("* Result for " + analysis.name())
 			# print(analysis.result())
 			# print(analysis.render())
-			analysis.result(store)
+			analysis.result(store, name=log_id)
 	if False:
 		for r in get_renderer(analyzers.LocomotionActionAnalyzer):
 			r().render(store.get_all())
@@ -118,8 +118,25 @@ if __name__ == '__main__':
 				writer.writerow(["name"] + [h.split(".")[-1] for h in headers])
 				for line in lines:
 					writer.writerow(line)
-
 	if True:
+		from datetime import datetime
+		json.dump(store.serializable(), open("simus.json", "w"), indent=2)
+		with open("simus.csv", "w") as csvfile:
+			csvfile.write("instanceconfig,log,simu,answered,universe_state,selected_actions,timestamp,time\n")
+			for key in store.get_store():
+				csvfile.write("{}\n".format(key))
+				for result in store.store[key]:
+					csvfile.write(",{}\n".format(result.name))
+					for i in result.get():
+						csvfile.write(",,{},{},{},{},{},{}\n".format(
+							i['answers']['@id'],
+							i['answers']['answered'],
+							len(i['answers']['universe_state']) if i['answers']['universe_state'] else 0,
+							len(i['selected_actions']) if i['selected_actions'] else 0,
+							i['timestamp'],
+							str(datetime.fromtimestamp(i['timestamp']/1000))
+						))
+	if False:
 		#json.dump(store.serializable(), open("new.json", "w"), indent=1)
 		from collections import defaultdict