diff --git a/analyzers/__init__.py b/analyzers/__init__.py index 4557f9d..b928374 100644 --- a/analyzers/__init__.py +++ b/analyzers/__init__.py @@ -2,7 +2,8 @@ from typing import List from .analyzer import Analyzer, Result from .analyzer.biogames import BoardDurationAnalyzer, SimulationRoundsAnalyzer, ActivationSequenceAnalyzer, \ - BiogamesCategorizer, ActivityMapper, BiogamesStore, InstanceConfig, SimulationOrderAnalyzer, SimulationCategorizer + BiogamesCategorizer, ActivityMapper, BiogamesStore, InstanceConfig, SimulationOrderAnalyzer, SimulationCategorizer, \ + SimulationFlagsAnalyzer from .analyzer.default import LogEntryCountAnalyzer, LocationAnalyzer, LogEntrySequenceAnalyzer, ActionSequenceAnalyzer, \ CategorizerStub, Store, ProgressAnalyzer from .analyzer.locomotion import LocomotionActionAnalyzer, CacheSequenceAnalyzer diff --git a/analyzers/analyzer/__init__.py b/analyzers/analyzer/__init__.py index e9f4e30..6b7d5e0 100644 --- a/analyzers/analyzer/__init__.py +++ b/analyzers/analyzer/__init__.py @@ -8,10 +8,11 @@ log: logging.Logger = logging.getLogger(__name__) class Result: - def __init__(self, analysis: Type, result: Sized): + def __init__(self, analysis: Type, result: Sized, name: str = None): self.result = result self.__analysis__ = analysis log.debug("set" + str(len(self.result))) + self.name = name def analysis(self): return self.__analysis__ @@ -21,7 +22,8 @@ class Result: return self.result def __repr__(self): - return "" + return "" class ResultStore: @@ -66,7 +68,11 @@ class ResultStore: def serializable(self): values = {} for key in self.store: - values[key] = [{"analysis": str(result.analysis()), "result": result.get()} for result in self.store[key]] + values[key] = [{ + "analysis": str(result.analysis()), + "result": result.get(), + "name": result.name + } for result in self.store[key]] return values @@ -84,7 +90,7 @@ class Analyzer: """ raise NotImplementedError() - def result(self, store: ResultStore) -> None: + def result(self, store: ResultStore, name=None) -> None: raise NotImplementedError() def name(self) -> str: diff --git a/analyzers/analyzer/biogames.py b/analyzers/analyzer/biogames.py index 96215e3..cbf9306 100644 --- a/analyzers/analyzer/biogames.py +++ b/analyzers/analyzer/biogames.py @@ -305,10 +305,7 @@ class SimulationOrderAnalyzer(Analyzer): class SimulationCategorizer(CategorizerStub): # TODO: refactor categorizer - __name__ = "SimulationCategorizer" - - def __init__(self, settings: LogSettings): - super().__init__(settings) + __name__ = "SimulationCategorizer"# TODO: rename -.- (InstanceConfigIDCategorizer) def process(self, entry: dict) -> bool: if self.key is "default": @@ -319,3 +316,23 @@ class SimulationCategorizer(CategorizerStub): # TODO: refactor categorizer print(entry) raise e return False + + +class SimulationFlagsAnalyzer(Analyzer): + __name__ = "SimuFlags" + + def __init__(self, settings: LogSettings) -> None: + super().__init__(settings) + self.store = [] + + def process(self, entry: dict) -> bool: + entry_type = entry[self.settings.type_field] + if entry_type in self.settings.custom['simulation_rounds']: + if entry["answers"][self.settings.type_field] in self.settings.custom["simu_data"]: + self.store.append(entry) + return False + + def result(self, store: ResultStore, name=None) -> None: + store.add(Result(type(self), self.store, name=name)) + + diff --git a/analyzers/analyzer/default.py b/analyzers/analyzer/default.py index bcb9d2a..9151313 100644 --- a/analyzers/analyzer/default.py +++ b/analyzers/analyzer/default.py @@ -89,7 +89,7 @@ class CategorizerStub(Analyzer): __name__ = "Categorizer" - def result(self, store: ResultStore) -> None: + def result(self, store: ResultStore, name=None) -> None: store.new_category(self.key) def __init__(self, settings: LogSettings): diff --git a/biogames2.json b/biogames2.json index 83072ea..7bedcb9 100644 --- a/biogames2.json +++ b/biogames2.json @@ -14,10 +14,11 @@ "analyzers": { "analyzers": [ "SimulationCategorizer", - "ActivityMapper" + "SimulationFlagsAnalyzer" ] }, "dis":[ + "ActivityMapper", "BiogamesCategorizer", "LogEntryCountAnalyzer", "SimulationOrderAnalyzer", @@ -70,8 +71,8 @@ "type": "Biogames", "url": "http://0.0.0.0:5000/game2/instance/log/list/", "login_url": "http://localhost:5000/game2/auth/json-login", - "username": "ba", - "password": "853451", + "username": "dev", + "password": "dev", "host":"http://0.0.0.0:5000" } } \ No newline at end of file diff --git a/filter_todo/pre_filter.py b/filter_todo/pre_filter.py new file mode 100644 index 0000000..27480c4 --- /dev/null +++ b/filter_todo/pre_filter.py @@ -0,0 +1,90 @@ +import os +from zipfile import ZipFile +import sqlite3 +import json +from collections import defaultdict + + +def get_json(filename): + log = [] + id = None + with ZipFile(filename) as zipf: + zipf.extract('instance_log.sqlite') + sql = sqlite3.connect('instance_log.sqlite') + cursor = sql.cursor() + for r in cursor.execute('SELECT json FROM log_entry;'): + entry = json.loads(r[0]) + log.append(entry) + if id is None: + id = entry['instance_id'] + sql.close() + os.remove('instance_log.sqlite') + return id, log + + +def is_finished(log): + for entry in log: + if "action" in entry: + if "LogEntryInstanceAction" in entry["@class"] and entry["action"][ + "@class"] == "de.findevielfalt.games.game2.instance.action.EndGameEnableAction" and entry['action'][ + 'enable']: + return True + return False + + +def get_simus(log): + simus = defaultdict(lambda: 0) + order = [] + actions = 0 + for entry in log: + if "LogEntryQuestion" in entry["@class"]: + if "SimulationBoardData" in entry["answers"]["@class"]: + id = entry["answers"]["@id"] + simus[id] += 1 + actions += 1 if entry['selected_actions'] else 0 + if not id in order: + order.append(id) + return dict(simus), order, actions + + +def simu_dist(simus): + dist = defaultdict(lambda: 0) + for instance in simus: + sim = simus[instance] + dist[len(sim)] += 1 + return dist + + +logs = {} +finished = [] +simus = {} +distribution = defaultdict(lambda: 0) +finished_and_simu = defaultdict(list) +files = {} +actions_dist = defaultdict(list) +with open('/home/agp8x/git/uni/ma/project/data/0000_ref') as src: + for line in src: + line = line.strip() + instance_id, log = get_json(line) + logs[instance_id] = log + files[instance_id] = line +for id in logs: + simus[id] = get_simus(logs[id]) + simu_count = len(simus[id][1]) + distribution[simu_count] += 1 + actions_dist[simus[id][2]].append(id) + if is_finished(logs[id]): + finished.append(id) + finished_and_simu[simu_count].append(id) +print("total: ", len(logs)) +print("finished: ", len(finished)) +print("simu_dist: ", len(distribution), json.dumps(distribution, sort_keys=True)) +for i in sorted(finished_and_simu): + print("fin+sim" + str(i) + ": ", len(finished_and_simu[i])) +for i in sorted(actions_dist): + print("actions: ", i, len(actions_dist[i])) +print(json.dumps(actions_dist[4], sort_keys=True, indent=2)) + +# print(finished_and_simu) +# for instance in finished_and_simu: +# print(files[instance]) diff --git a/log_analyzer.py b/log_analyzer.py index c0b0723..9b773e0 100644 --- a/log_analyzer.py +++ b/log_analyzer.py @@ -63,7 +63,7 @@ if __name__ == '__main__': # "fec57041458e6cef98652df625", ] log_ids = [] # with open("/home/clemens/git/ma/test/filtered") as src: - with open("/home/clemens/git/ma/test/filtered") as src: + with open("/home/agp8x/git/uni/ma/project/data/0000_ref") as src: for line in src: line = line.strip() log_ids.append(line) @@ -73,7 +73,7 @@ if __name__ == '__main__': log.info("* Result for " + analysis.name()) # print(analysis.result()) # print(analysis.render()) - analysis.result(store) + analysis.result(store, name=log_id) if False: for r in get_renderer(analyzers.LocomotionActionAnalyzer): r().render(store.get_all()) @@ -118,8 +118,25 @@ if __name__ == '__main__': writer.writerow(["name"] + [h.split(".")[-1] for h in headers]) for line in lines: writer.writerow(line) - if True: + from datetime import datetime + json.dump(store.serializable(), open("simus.json", "w"), indent=2) + with open("simus.csv", "w") as csvfile: + csvfile.write("instanceconfig,log,simu,answered,universe_state,selected_actions,timestamp,time\n") + for key in store.get_store(): + csvfile.write("{}\n".format(key)) + for result in store.store[key]: + csvfile.write(",{}\n".format(result.name)) + for i in result.get(): + csvfile.write(",,{},{},{},{},{},{}\n".format( + i['answers']['@id'], + i['answers']['answered'], + len(i['answers']['universe_state']) if i['answers']['universe_state'] else 0, + len(i['selected_actions']) if i['selected_actions'] else 0, + i['timestamp'], + str(datetime.fromtimestamp(i['timestamp']/1000)) + )) + if False: #json.dump(store.serializable(), open("new.json", "w"), indent=1) from collections import defaultdict