diff --git a/.github/workflows/test_pytest.yml b/.github/workflows/test_pytest.yml index a990cbfd2..a40d0d3cd 100644 --- a/.github/workflows/test_pytest.yml +++ b/.github/workflows/test_pytest.yml @@ -138,5 +138,5 @@ jobs: - name: Run ci_engine session run: | cd contrib - python ci_engine.py -n 5 + python ci_engine.py run -n 5 timeout-minutes: 5 diff --git a/contrib/ci_engine.py b/contrib/ci_engine.py index bbbac8e4c..ced594587 100755 --- a/contrib/ci_engine.py +++ b/contrib/ci_engine.py @@ -41,18 +41,28 @@ """ - +import argparse +import asyncio +import collections +from concurrent.futures import ThreadPoolExecutor import configparser import itertools import json import logging +import operator +from pathlib import Path +import queue +import shlex +import signal import sqlite3 -import subprocess import sys +import threading +from tempfile import TemporaryDirectory from random import Random -import click from rich.console import Console +from rich.progress import (BarColumn, MofNCompleteColumn, Progress, + SpinnerColumn, Task, TextColumn, TimeElapsedColumn) from rich.table import Table from pelita.network import RemotePlayerFailure @@ -64,25 +74,40 @@ # the path of the configuration file CFG_FILE = './ci.cfg' -def hash_team(team_spec): +EXIT = threading.Event() + +def signal_handler(_signal, _frame): + _logger.warning('Program terminated by kill or ctrl-c') + EXIT.set() + sys.exit() + +signal.signal(signal.SIGINT, signal_handler) + +async def hash_team(team_spec, semaphore): external_call = [sys.executable, '-m', 'pelita.scripts.pelita_player', 'hash-team', team_spec] - _logger.debug("Executing: %r", external_call) - res = subprocess.run(external_call, capture_output=True, text=True) - return res.stdout.strip().split("\n")[-1].strip() + async with semaphore: + _logger.debug("Executing: %r", shlex.join(external_call)) + proc = await asyncio.create_subprocess_exec(*external_call, + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE + ) + stdout, stderr = await proc.communicate() + + return stdout.decode().strip().split("\n")[-1].strip() class CI_Engine: """Continuous Integration Engine.""" def __init__(self, cfgfile): - self.players = [] + self.players = {} config = configparser.ConfigParser() config.read_file(cfgfile) for name, path in config.items('agents'): - self.players.append({'name': name, 'path': path}) + self.players[name]= {'path': path} self.rounds = config['general'].getint('rounds', None) self.size = config['general'].get('size', None) @@ -92,44 +117,68 @@ def __init__(self, cfgfile): self.db_file = config.get('general', 'db_file') self.dbwrapper = DB_Wrapper(self.db_file) - def load_players(self): + def load_players(self, concurrency=1): hash_cache = {} # remove players from db which are not in the config anymore for pname in self.dbwrapper.get_players(): - if pname not in [p['name'] for p in self.players]: + if pname not in self.players: _logger.debug('Removing %s from database, because it is not among the current players.' % (pname)) self.dbwrapper.remove_player(pname) + semaphore = asyncio.Semaphore(concurrency) + + async def do_hash(): + players = [(pname, player['path']) for pname, player in self.players.items()] + tasks = [asyncio.create_task(hash_team(player[1], semaphore)) for player in players] + hashes = await asyncio.gather(*tasks) + return {player[0]: hash for (player, hash) in zip(players, hashes)} + + hash_cache = asyncio.run(do_hash()) + # add new players into db - for player in self.players: - pname, path = player['name'], player['path'] + for pname, player in self.players.items(): + path = player['path'] if pname not in self.dbwrapper.get_players(): _logger.debug('Adding %s to database.' % pname) - hash_cache[path] = hash_team(path) - self.dbwrapper.add_player(pname, hash_cache[path]) + self.dbwrapper.add_player(pname, hash_cache[pname]) # reset players where the directory hash changed - for player in self.players: + for pname, player in self.players.items(): path = player['path'] - pname = player['name'] - new_hash = hash_cache.get(path, hash_team(path)) + new_hash = hash_cache[pname] if new_hash != self.dbwrapper.get_player_hash(pname): _logger.debug('Resetting %s because its module hash changed.' % pname) self.dbwrapper.remove_player(pname) self.dbwrapper.add_player(pname, new_hash) - for player in self.players: - path = player['path'] - pname = player['name'] + def check_team_name(args): + pname, path = args try: _logger.debug('Querying team name for %s.' % pname) - team_name = check_team(player['path']) - self.dbwrapper.add_team_name(pname, team_name) + team_name = check_team(path, timeout=6*concurrency) + return { 'team_name': team_name } except RemotePlayerFailure as e: e_type, e_msg = e.args - _logger.debug(f'Could not import {player} at path {path} ({e_type}): {e_msg}') - player['error'] = e.args + _logger.debug(f'Could not import {pname} at path {path} ({e_type}): {e_msg}') + return { 'error': e.args } + + with ThreadPoolExecutor(max_workers=concurrency) as executor: + players = [(pname, player['path']) for pname, player in self.players.items()] + team_names = executor.map(check_team_name, players) + + for (pname, path), team_name in zip(players, team_names): + if 'error' in team_name: + self.players[pname]['error'] = team_name['error'] + else: + self.dbwrapper.add_team_name(pname, team_name['team_name']) + + for pname in self.players: + if 'error' in self.players[pname]: + print(pname, self.players[pname]) + else: + print(pname, self.players[pname], self.dbwrapper.get_team_name(pname)) + def run_game(self, p1, p2): """Run a single game. @@ -143,32 +192,49 @@ def run_game(self, p1, p2): the indices of the players """ - team_specs = [self.players[i]['path'] for i in (p1, p2)] - print(f"Playing {self.players[p1]['name']} against {self.players[p2]['name']}.") + team_specs = [self.players[p1]['path'], self.players[p2]['path']] - final_state, stdout, stderr = call_pelita(team_specs, - rounds=self.rounds, - size=self.size, - viewer=self.viewer, - seed=self.seed) + with TemporaryDirectory() as tmpdir: - if final_state['whowins'] == 2: - result = -1 - else: - result = final_state['whowins'] + final_state, stdout, stderr = call_pelita(team_specs, + rounds=self.rounds, + size=self.size, + viewer=self.viewer, + seed=self.seed, + store_output=tmpdir, + timeout=10, + initial_timeout=120, + exit_flag=EXIT + ) - del final_state['walls'] - del final_state['food'] + if not final_state: + res = (p1, p2, None, final_state, stdout, stderr) + return res - _logger.info('Final state: %r', final_state) - _logger.debug('Stdout: %r', stdout) - if stderr: - _logger.warning('Stderr: %r', stderr) - p1_name, p2_name = self.players[p1]['name'], self.players[p2]['name'] - self.dbwrapper.add_gameresult(p1_name, p2_name, result, final_state, stdout, stderr) + if final_state['whowins'] == 2: + result = -1 + else: + result = final_state['whowins'] + del final_state['walls'] + del final_state['food'] - def start(self, n): + _logger.info('Final state: %r', final_state) + _logger.debug('Stdout: %r', stdout) + if stderr: + _logger.warning('Stderr: %r', stderr) + + p1_stdout = (Path(tmpdir) / 'blue.out').read_text() + p1_stderr = (Path(tmpdir) / 'blue.err').read_text() + + p2_stdout = (Path(tmpdir) / 'red.out').read_text() + p2_stderr = (Path(tmpdir) / 'red.err').read_text() + + res = (p1, p2, result, final_state, [stdout, stderr], [p1_stdout, p1_stderr], [p2_stdout, p2_stderr]) + return res + + + def start(self, n, thread_count): """Start the Engine. This method will start and infinite loop, testing each agent @@ -183,27 +249,107 @@ def start(self, n): >>> ci.start() """ - loop = itertools.repeat(None) if n == 0 else itertools.repeat(None, n) - rng = Random() - - for _ in loop: - # choose the player with the least number of played game, - # match with another random player - # mix the sides and let them play - broken_players = {idx for idx, player in enumerate(self.players) if player.get('error')} - game_count = [(self.dbwrapper.get_game_count(p['name']), idx) for idx, p in enumerate(self.players)] - players_sorted = [idx for count, idx in sorted(game_count) if idx not in broken_players] - a, rest = players_sorted[0], players_sorted[1:] - b = rng.choice(rest) - players = [a, b] - rng.shuffle(players) - - self.run_game(players[0], players[1]) - self.pretty_print_results(highlight=[self.players[players[0]]['name'], self.players[players[1]]['name']]) - print('------------------------------') - - - def get_results(self, idx, idx2=None): + + with Progress( + SpinnerColumn(), + TextColumn("[progress.description]{task.description}"), + BarColumn(), + MofNCompleteColumn(), + TimeElapsedColumn(), + # transient=True + ) as progress: + + loop = itertools.repeat(None) if n == 0 else itertools.repeat(None, n) + rng = Random() + + game_counts = self.dbwrapper.get_game_counts() + + for pname, player in self.players.items(): + if "error" in player and pname in game_counts: + del game_counts[pname] + + def worker(q, r, lock=threading.Lock()): + for task in iter(q.get, None): # blocking get until None is received + try: + count, slf, p1, p2 = task + + with lock: + progress_task = progress.add_task(f"Playing #{count}: {p1} against {p2}.") + + res = slf.run_game(p1, p2) + + with lock: + progress.update(progress_task, completed=True, visible=False) + + r.put((count, (p1, p2), res)) + + finally: + q.task_done() + + worker_count = thread_count + q = queue.Queue(maxsize=thread_count) + r = queue.Queue() + threads = [threading.Thread(target=worker, args=[q, r], daemon=False) + for _ in range(worker_count)] + for t in threads: + t.start() + + for count, _ in enumerate(loop): + # choose the player with the least number of played game, + # match with another random player + # mix the sides and let them play + + players_sorted = sorted(list(game_counts.items()), key=operator.itemgetter(1)) + + a = players_sorted[0][0] + b = rng.choice(players_sorted[1:])[0] + + players = [a, b] + rng.shuffle(players) + + q.put((count, self, players[0], players[1])) + + game_counts[a] += 1 + game_counts[b] += 1 + + try: + count, players, res = r.get_nowait() + final_state = res[3] + if final_state: + progress.console.print(f"Storing #{count}: {players[0]} against {players[1]}.") + else: + progress.console.print(f"Not storing #{count}: {players[0]} against {players[1]}.") + self.dbwrapper.add_gameresult(*res) + except queue.Empty: + pass + except Exception: + pass + + if EXIT.is_set(): + break + + q.join() # block until all spawned tasks are done + + while True: + try: + count, players, res = r.get_nowait() + final_state = res[3] + if final_state: + progress.console.print(f"Storing #{count}: {players[0]} against {players[1]}.") + else: + progress.console.print(f"Not storing #{count}: {players[0]} against {players[1]}.") + self.dbwrapper.add_gameresult(*res) + except queue.Empty: + break + + for _ in threads: # signal workers to quit + q.put(None) + + for t in threads: # wait until workers exit + t.join() + + + def get_results(self, p1_name, p2_name=None): """Get the results so far. This method goes through the internal list of of all game @@ -245,18 +391,16 @@ def get_results(self, idx, idx2=None): """ win, loss, draw = 0, 0, 0 - p1_name = self.players[idx]['name'] - p2_name = None if idx2 is None else self.players[idx2]['name'] relevant_results = self.dbwrapper.get_results(p1_name, p2_name) for p1, p2, r in relevant_results: - if (idx2 is None and p1_name == p1) or (idx2 is not None and p1_name == p1 and p2_name == p2): + if (p2_name is None and p1_name == p1) or (p2_name is not None and p1_name == p1 and p2_name == p2): if r == 0: win += 1 elif r == 1: loss += 1 elif r == -1: draw += 1 - if (idx2 is None and p1_name == p2) or (idx2 is not None and p1_name == p2 and p2_name == p1): + if (p2_name is None and p1_name == p2) or (p2_name is not None and p1_name == p2 and p2_name == p1): if r == 1: win += 1 elif r == 0: @@ -265,7 +409,7 @@ def get_results(self, idx, idx2=None): draw += 1 return win, loss, draw - def get_errorcount(self, idx): + def get_errorcount(self, p_name): """Gets the error count for team idx Parameters @@ -279,17 +423,15 @@ def get_errorcount(self, idx): the number of errors for this player """ - p_name = self.players[idx]['name'] error_count, fatalerror_count = self.dbwrapper.get_errorcount(p_name) return error_count, fatalerror_count - def get_team_name(self, idx): + def get_team_name(self, p_name): """Get last registered team name. team_name : string """ - p_name = self.players[idx]['name'] return self.dbwrapper.get_team_name(p_name) def gen_elo(self): @@ -300,104 +442,38 @@ def elo_change(a, b, outcome): return k * (outcome - expected) from collections import defaultdict - elo = defaultdict(lambda: 1500) + elo = defaultdict(lambda: 1500.) g = self.dbwrapper.cursor.execute(""" SELECT player1, player2, result FROM games """).fetchall() for p1, p2, result in g: + change = 0 if result == 0: change = elo_change(elo[p1], elo[p2], 1) if result == 1: change = elo_change(elo[p1], elo[p2], 0) if result == -1: change = elo_change(elo[p1], elo[p2], 0.5) + elo[p1] += change elo[p2] -= change return elo - def pretty_print_results(self, highlight=None): + def pretty_print_results(self, full=False, team=None, highlight=None): """Pretty print the current results. """ if highlight is None: highlight = [] - console = Console() - # Some guesswork in here - MAX_COLUMNS = (console.width - 40) // 12 - if MAX_COLUMNS < 4: - # Let’s be honest: You should enlarge your terminal window even before that - MAX_COLUMNS = 4 - - res = self.dbwrapper.get_wins_losses() - rows = { k: list(v) for k, v in itertools.groupby(res, key=lambda x:x[0]) } - - good_players = [p for p in self.players if not p.get('error')] - bad_players = [p for p in self.players if p.get('error')] - - num_rows_per_player = (len(good_players) // MAX_COLUMNS) + 1 - row_style = [*([""] * num_rows_per_player), *(["dim"] * num_rows_per_player)] - - table = Table(row_styles=row_style, title="Cross results") - table.add_column("") - table.add_column("Name") - table.add_column("Score", justify="right") - table.add_column("W/D/L") + good_players = [p for p, player in self.players.items() if not player.get('error')] + bad_players = [p for p, player in self.players.items() if player.get('error')] - column_players = [[] for _idx in range(min(MAX_COLUMNS, len(good_players)))] - # if we have more good_players than allowed columns, we must wrap around - for idx, _p in enumerate(good_players): - column_players[idx % MAX_COLUMNS].append(idx) - - for midx in column_players: - table.add_column('\n'.join(map(str, midx))) - - - def batched(iterable, n): - # Backport from Python 3.12 - # batched('ABCDEFG', 3) → ABC DEF G - if n < 1: - raise ValueError('n must be at least one') - iterator = iter(iterable) - while batch := tuple(itertools.islice(iterator, n)): - yield batch - - result = [] - for idx, p in enumerate(good_players): - win, loss, draw = self.get_results(idx) - error_count, fatalerror_count = self.get_errorcount(idx) - try: - team_name = self.get_team_name(idx) - except ValueError: - team_name = None - score = 0 if (win+loss+draw) == 0 else (win-loss) / (win+loss+draw) - result.append([score, win, draw, loss, p['name'], team_name, error_count, fatalerror_count]) - wdl = f"{win:3d},{draw:3d},{loss:3d}" - - try: - row = rows[p['name']] - except KeyError: - continue - vals = { k: (w,l,d) for _p1, k, w, l, d in row } - - cross_results = [] - for idx2, p2 in enumerate(good_players): - win, loss, draw = vals.get(p2['name'], (0, 0, 0)) - if idx == idx2: - cross_results.append(" - - - ") - else: - cross_results.append(f"{win:2d},{draw:2d},{loss:2d}") - - for c, r in enumerate(batched(cross_results, MAX_COLUMNS)): - if c == 0: - table.add_row(f"{idx}", p['name'], f"{score:.2f}", wdl, *r) - else: - table.add_row("", "", "", "", *r) + console = Console() - console.print(table) table = Table(title="Bot ranking") @@ -408,10 +484,22 @@ def batched(iterable, n): table.add_column("# Losses") table.add_column("Score") table.add_column("ELO") - table.add_column("Error count") + table.add_column("# Timeouts") table.add_column("# Fatal Errors") - elo = self.gen_elo() + elo = dict(self.dbwrapper.get_elo()) + # elo = self.gen_elo() + + result = [] + for idx, pname in enumerate(good_players): + win, loss, draw = self.get_results(pname) + error_count, fatalerror_count = self.get_errorcount(pname) + try: + team_name = self.get_team_name(pname) + except ValueError: + team_name = None + score = 0 if (win+loss+draw) == 0 else (win-loss) / (win+loss+draw) + result.append([score, win, draw, loss, pname, team_name, error_count, fatalerror_count]) result.sort(reverse=True) for [score, win, draw, loss, name, team_name, error_count, fatalerror_count] in result: @@ -424,7 +512,7 @@ def batched(iterable, n): f"{draw}", f"{loss}", f"{score:6.3f}", - f"{elo[name]: >4.0f}", + f"{elo.get(name, 0): >4.0f}", f"{error_count}", f"{fatalerror_count}", style=style, @@ -433,7 +521,121 @@ def batched(iterable, n): console.print(table) for p in bad_players: - print("% 30s ***%30s***" % (p['name'], p['error'])) + print("% 30s ***%30s***" % (p, self.players[p]['error'])) + + + if full: + # Some guesswork in here + MAX_COLUMNS = (console.width - 40) // 12 + if MAX_COLUMNS < 4: + # Let’s be honest: You should enlarge your terminal window even before that + MAX_COLUMNS = 4 + + res = self.dbwrapper.get_wins_losses() + rows = { k: list(v) for k, v in itertools.groupby(res, key=lambda x:x[0]) } + + num_rows_per_player = (len(good_players) // MAX_COLUMNS) + 1 + row_style = [*([""] * num_rows_per_player), *(["dim"] * num_rows_per_player)] + + table = Table(row_styles=row_style, title="Cross results") + table.add_column("") + table.add_column("Name") + table.add_column("Score", justify="right") + table.add_column("W/D/L") + + column_players = [[] for _idx in range(min(MAX_COLUMNS, len(good_players)))] + # if we have more good_players than allowed columns, we must wrap around + for idx, _p in enumerate(good_players): + column_players[idx % MAX_COLUMNS].append(idx) + + for midx in column_players: + table.add_column('\n'.join(map(str, midx))) + + + def batched(iterable, n): + # Backport from Python 3.12 + # batched('ABCDEFG', 3) → ABC DEF G + if n < 1: + raise ValueError('n must be at least one') + iterator = iter(iterable) + while batch := tuple(itertools.islice(iterator, n)): + yield batch + + for idx, pname in enumerate(good_players): + win, loss, draw = self.get_results(pname) + error_count, fatalerror_count = self.get_errorcount(pname) + try: + team_name = self.get_team_name(pname) + except ValueError: + team_name = None + score = 0 if (win+loss+draw) == 0 else (win-loss) / (win+loss+draw) + wdl = f"{win:3d},{draw:3d},{loss:3d}" + + try: + row = rows[pname] + except KeyError: + continue + vals = { k: (w,l,d) for _p1, k, w, l, d in row } + + cross_results = [] + for idx2, p2name in enumerate(good_players): + win, loss, draw = vals.get(p2name, (0, 0, 0)) + if idx == idx2: + cross_results.append(" - - - ") + else: + cross_results.append(f"{win:2d},{draw:2d},{loss:2d}") + + for c, r in enumerate(batched(cross_results, MAX_COLUMNS)): + if c == 0: + table.add_row(f"{idx}", pname, f"{score:.2f}", wdl, *r) + else: + table.add_row("", "", "", "", *r) + + console.print(table) + + elif team: + MAX_COLUMNS = (console.width - 40) // 12 + if MAX_COLUMNS < 4: + # Let’s be honest: You should enlarge your terminal window even before that + MAX_COLUMNS = 4 + + res = self.dbwrapper.get_wins_losses(team=team) + rows = {k: list(v) for k, v in itertools.groupby(res, key=lambda x:x[1])} + + row_style = ["", "dim"] + + table = Table(row_styles=row_style, title=f"Match results for team {team}") + table.add_column("Name") + table.add_column("# Matches") + table.add_column("# Wins") + table.add_column("# Draws") + table.add_column("# Losses") + + for idx, pname in enumerate(good_players): + try: + team_name = self.get_team_name(pname) + except ValueError: + team_name = None + + try: + row = rows[pname] + except KeyError: + continue + + for r in row: # there should only be one row + p1, p2, win, loss, draw = r + + display_name = f"{pname} ({team_name})" if team_name else f"{pname}" + + table.add_row( + display_name, + f"{win+draw+loss}", + f"{win}", + f"{draw}", + f"{loss}", + ) + + console.print(table) class DB_Wrapper: @@ -473,10 +675,22 @@ def create_tables(self): """) self.cursor.execute(""" CREATE TABLE IF NOT EXISTS games - (player1 text, player2 text, result int, final_state text, stdout text, stderr text, + ( + id INTEGER PRIMARY KEY, + player1 text, player2 text, result int, final_state text, + player1_timeouts int, player2_timeouts int, + player1_fatal_errors int, player2_fatal_errors int, FOREIGN KEY(player1) REFERENCES players(name) ON DELETE CASCADE, FOREIGN KEY(player2) REFERENCES players(name) ON DELETE CASCADE) """) + self.cursor.execute(""" + CREATE TABLE IF NOT EXISTS game_output + (game_id int, + stdout text, stderr text, + player1_stdout text, player1_stderr text, + player2_stdout text, player2_stderr text, + FOREIGN KEY(game_id) REFERENCES games(id) ON DELETE CASCADE) + """) self.connection.commit() def get_players(self): @@ -567,7 +781,7 @@ def remove_player(self, pname): WHERE name = ?""", (pname,)) self.connection.commit() - def add_gameresult(self, p1_name, p2_name, result, final_state, std_out, std_err): + def add_gameresult(self, p1_name, p2_name, result, final_state, std, p1_out, p2_out): """Add a new game result to the database. Parameters @@ -582,10 +796,30 @@ def add_gameresult(self, p1_name, p2_name, result, final_state, std_out, std_err STDOUT and STDERR of the game """ + + stdout, stderr = std + p1_stdout, p1_stderr = p1_out + p2_stdout, p2_stderr = p2_out + + if not final_state: + return self.cursor.execute(""" INSERT INTO games - VALUES (?, ?, ?, ?, ?, ?) - """, [p1_name, p2_name, result, json.dumps(final_state), std_out, std_err]) + (player1, player2, result, final_state, + player1_timeouts, player2_timeouts, + player1_fatal_errors, player2_fatal_errors) + VALUES (?, ?, ?, ?, ?, ?, ?, ?) + RETURNING id + """, [p1_name, p2_name, result, json.dumps(final_state), + final_state['num_errors'][0], final_state['num_errors'][1], + len(final_state['fatal_errors'][0]), len(final_state['fatal_errors'][1])]) + game_id, = self.cursor.fetchone() + self.cursor.execute(""" + INSERT INTO game_output + VALUES (?, ?, ?, ?, ?, ?, ?) + """, [game_id, + stdout, stderr, + p1_stdout, p1_stderr, p2_stdout, p2_stderr]) self.connection.commit() def get_results(self, p1_name, p2_name=None): @@ -640,6 +874,32 @@ def get_team_name(self, p_name): raise ValueError('Player %s does not exist in database.' % p_name) return res[0] + def get_game_counts(self): + """Get number of games per player. + + Returns + ------- + relevant_results : dict[name, int] + + """ + self.cursor.execute(""" + SELECT p.name, COUNT(g.player) AS num_games + FROM + players p + LEFT JOIN + ( + SELECT player1 AS player FROM games + UNION ALL + SELECT player2 AS player FROM games + ) g + ON p.name = g.player + GROUP BY p.name + """) + counts = collections.Counter() + for name, val in self.cursor.fetchall(): + counts[name] += val + return counts + def get_game_count(self, p1_name, p2_name=None): """Get number of games involving player1 (AND player2 if specified). @@ -681,45 +941,29 @@ def get_errorcount(self, p1_name): Returns ------- error_count, fatalerror_count : errorcount - """ self.cursor.execute(""" - SELECT sum(c) FROM + SELECT sum(timeouts), sum(fatal_errors) FROM ( - SELECT sum(json_extract(final_state, '$.num_errors[0]')) AS c + SELECT + sum(player1_timeouts) AS timeouts, + sum(player1_fatal_errors) AS fatal_errors FROM games WHERE player1 = :p1 UNION ALL - SELECT sum(json_extract(final_state, '$.num_errors[1]')) AS c + SELECT + sum(player2_timeouts) AS timeouts, + sum(player2_fatal_errors) AS fatal_errors FROM games WHERE player2 = :p1 ) """, dict(p1=p1_name)) - error_count, = self.cursor.fetchone() + timeouts, fatal_errorcount = self.cursor.fetchone() - self.cursor.execute(""" - SELECT sum(c) FROM - ( - SELECT count(*) AS c - FROM games - WHERE player1 = :p1 AND - json_extract(final_state, '$.fatal_errors[0]') != '[]' - - UNION ALL - - SELECT count(*) AS c - FROM games - WHERE player2 = :p1 AND - json_extract(final_state, '$.fatal_errors[1]') != '[]' - ) - """, - dict(p1=p1_name)) - fatal_errorcount, = self.cursor.fetchone() - - return error_count, fatal_errorcount + return timeouts, fatal_errorcount def get_wins_losses(self, team=None): """ Get all wins and losses combined in a table of @@ -800,31 +1044,131 @@ def get_wins_losses(self, team=None): return self.cursor.execute(query).fetchall() -@click.command() -@click.option('--log', - is_flag=False, flag_value="-", default=None, metavar='LOGFILE', - help="print debugging log information to LOGFILE (default 'stderr')") -@click.option('--config', - default=CFG_FILE, - type=click.File('r'), - help='Configuration file') -@click.option('-n', help='run N times', type=int, default=0) -@click.option('--print', is_flag=True, default=False, - help='Print scores and exit.') -@click.option('--nohash', is_flag=True, default=False, - help='Do not hash the players') -def main(log, config, n, print, nohash): - if log is not None: - start_logging(log, __name__) - start_logging(log, 'pelita') - - ci_engine = CI_Engine(config) - if print: - ci_engine.pretty_print_results() - else: - if not nohash: - ci_engine.load_players() - ci_engine.start(n) + def get_elo(self): + query = """ + WITH RECURSIVE + ordered_matches AS ( + SELECT + ROW_NUMBER() OVER (ORDER BY rowid) AS match_num, + player1, + player2, + result + FROM games + ), + + -- Initialize with first match + elo_recursive(match_num, player1, player2, result, + rating1, rating2, + rating_json) AS ( + SELECT + match_num, + player1, + player2, + result, + 1500.0, + 1500.0, + json_object(player1, 1500.0, player2, 1500.0) + FROM ordered_matches + WHERE match_num = 1 + + UNION ALL + + SELECT + om.match_num, + om.player1, + om.player2, + om.result, + + -- Get ratings from JSON state + IFNULL(CAST(json_extract(er.rating_json, '$.' || om.player1) AS REAL), 1500.0), + IFNULL(CAST(json_extract(er.rating_json, '$.' || om.player2) AS REAL), 1500.0), + + -- Update JSON state with new ratings + json_set( + er.rating_json, + '$.' || om.player1, + ROUND( + IFNULL(CAST(json_extract(er.rating_json, '$.' || om.player1) AS REAL), 1500.0) + + 32 * ((CASE om.result WHEN 0 THEN 1.0 WHEN -1 THEN 0.5 ELSE 0.0 END) - + 1.0 / (1 + pow(10, ( + IFNULL(CAST(json_extract(er.rating_json, '$.' || om.player2) AS REAL), 1500.0) - + IFNULL(CAST(json_extract(er.rating_json, '$.' || om.player1) AS REAL), 1500.0) + ) / 400.0))), 2), + '$.' || om.player2, + ROUND( + IFNULL(CAST(json_extract(er.rating_json, '$.' || om.player2) AS REAL), 1500.0) + + 32 * ((CASE om.result WHEN 0 THEN 0.0 WHEN -1 THEN 0.5 ELSE 1.0 END) - + 1.0 / (1 + pow(10, ( + IFNULL(CAST(json_extract(er.rating_json, '$.' || om.player1) AS REAL), 1500.0) - + IFNULL(CAST(json_extract(er.rating_json, '$.' || om.player2) AS REAL), 1500.0) + ) / 400.0))), 2) + ) + FROM ordered_matches om + JOIN elo_recursive er ON om.match_num = er.match_num + 1 + ), + + final AS ( + SELECT rating_json + FROM elo_recursive + ORDER BY match_num DESC + LIMIT 1 + ) + SELECT + key AS player, + ROUND(value, 2) AS rating + FROM final, json_each(rating_json) + ORDER BY rating DESC; + + """ + return self.cursor.execute(query).fetchall() + +def run(args): + with open(args.config) as f: + ci_engine = CI_Engine(f) + if not args.no_hash: + ci_engine.load_players(concurrency=args.thread_count) + ci_engine.start(args.n, args.thread_count) + +def print_scores(args): + with open(args.config) as f: + ci_engine = CI_Engine(f) + ci_engine.pretty_print_results(full=args.full, team=args.team) + +def hash_teams(args): + with open(args.config) as f: + ci_engine = CI_Engine(f) + ci_engine.load_players(concurrency=args.thread_count) + if __name__ == '__main__': - main() + parser = argparse.ArgumentParser() + parser.add_argument('--log', help="Print debugging log information to LOGFILE (default 'stderr').", + metavar='LOGFILE', const='-', nargs='?') + parser.add_argument('--config', help="Print debugging log information to LOGFILE (default 'stderr').", + metavar='FILE', default=CFG_FILE) + + subparsers = parser.add_subparsers(required=True) + + parser_run = subparsers.add_parser('run') + parser_run.add_argument('-n', help='run N times', type=int, default=0) + parser_run.add_argument('--thread-count', '-t', help='run in parallel', type=int, default=1) + parser_run.add_argument('--no-hash', help='Do not hash the players prior to running', action='store_true', default=False) + parser_run.set_defaults(func=run) + + parser_print_scores = subparsers.add_parser('print-scores') + full_or_team = parser_print_scores.add_mutually_exclusive_group() + parser_print_scores.add_argument('--full', help='show full pair statistics', action='store_true', default=False) + parser_print_scores.add_argument('--team', help='show statistics for team', type=str, default=None) + parser_print_scores.set_defaults(func=print_scores) + + parser_hash = subparsers.add_parser('hash-teams') + parser_hash.set_defaults(func=hash_teams) + parser_hash.add_argument('--thread-count', '-t', help='run in parallel', type=int, default=1) + + args = parser.parse_args() + + if args.log is not None: + start_logging(args.log, __name__) + start_logging(args.log, 'pelita') + + args.func(args) diff --git a/contrib/test_ci_engine.py b/contrib/test_ci_engine.py index 2e59e5218..22f5a24ef 100644 --- a/contrib/test_ci_engine.py +++ b/contrib/test_ci_engine.py @@ -137,3 +137,5 @@ def test_wins_losses(db_wrapper): assert db_wrapper.get_game_count('p1', 'p2') == 3 assert db_wrapper.get_game_count('p2', 'p1') == 3 assert db_wrapper.get_game_count('p3', 'p1') == 1 + + assert db_wrapper.get_game_counts() == dict(p1=4, p2=3, p3=1) diff --git a/pelita/game.py b/pelita/game.py index 2bd7c217b..1da7b5bc6 100644 --- a/pelita/game.py +++ b/pelita/game.py @@ -3,6 +3,7 @@ import logging import math import os +import shlex import subprocess import sys import time @@ -93,7 +94,7 @@ def _run_external_viewer(self, subscribe_sock, controller, geometry, delay, stop external_call = [sys.executable, '-m', tkviewer] + viewer_args - _logger.debug("Executing: %r", external_call) + _logger.debug("Executing: %r", shlex.join(external_call)) # os.setsid will keep the viewer from closing when the main process exits # a better solution might be to decouple the viewer from the main process if _mswindows: diff --git a/pelita/scripts/pelita_server.py b/pelita/scripts/pelita_server.py index 3db613a1a..2591bc97d 100755 --- a/pelita/scripts/pelita_server.py +++ b/pelita/scripts/pelita_server.py @@ -506,7 +506,7 @@ def play_remote(team_spec, pair_addr, silent_bots=False): pair_addr, *(['--silent-bots'] if silent_bots else []), ] - _logger.debug("Executing: %r", external_call) + _logger.debug("Executing: %r", shlex.join(external_call)) sub = subprocess.Popen(external_call) return sub @@ -517,7 +517,7 @@ def _check_team(team_spec): 'pelita.scripts.pelita_player', 'check-team', team_spec] - _logger.debug("Executing: %r", external_call) + _logger.debug("Executing: %r", shlex.join(external_call)) res = subprocess.run(external_call, capture_output=True, text=True) return res.stdout.strip() diff --git a/pelita/scripts/pelita_tournament.py b/pelita/scripts/pelita_tournament.py index 7515561c1..e53635da5 100755 --- a/pelita/scripts/pelita_tournament.py +++ b/pelita/scripts/pelita_tournament.py @@ -37,29 +37,6 @@ def firstNN(*args): """ return next(filter(lambda x: x is not None, args), None) - -def shlex_unsplit(cmd): - """ - Translates a list of command arguments into bash-like ‘human’ readable form. - Pseudo-reverses shlex.split() - - Example - ------- - >>> shlex_unsplit(["command", "-f", "Hello World"]) - "command -f 'Hello World'" - - Parameters - ---------- - cmd : list of string - command + parameter list - - Returns - ------- - string - """ - return " ".join(shlex.quote(arg) for arg in cmd) - - def create_directory(prefix): for suffix in itertools.count(0): path = Path('{}-{:02d}'.format(prefix, suffix)) @@ -139,9 +116,9 @@ def setup(): print("Please enter the location of the sound-giving binary:") sound_path = input() elif res == "s": - sound_path = shlex_unsplit(sound["say"]) + sound_path = shlex.join(sound["say"]) elif res == "f": - sound_path = shlex_unsplit(sound["flite"]) + sound_path = shlex.join(sound["flite"]) else: continue diff --git a/pelita/team.py b/pelita/team.py index 583793bf3..1ea79ea65 100644 --- a/pelita/team.py +++ b/pelita/team.py @@ -1,6 +1,7 @@ import logging import os +import shlex import subprocess import sys import time @@ -464,7 +465,7 @@ def _call_pelita_player(self, team_spec, address, color='', store_output=False): team_spec, address] - _logger.debug("Executing: %r", external_call) + _logger.debug("Executing: %r", shlex.join(external_call)) if store_output == subprocess.DEVNULL: return (subprocess.Popen(external_call, stdout=store_output), None, None) elif store_output: diff --git a/pelita/tournament/__init__.py b/pelita/tournament/__init__.py index a2193edac..decd9a27c 100644 --- a/pelita/tournament/__init__.py +++ b/pelita/tournament/__init__.py @@ -98,7 +98,8 @@ def run_and_terminate_process(args, **kwargs): p.kill() -def call_pelita(team_specs, *, rounds, size, viewer, seed, team_infos=None, write_replay=False, store_output=False): +def call_pelita(team_specs, *, rounds, size, viewer, seed, timeout=3, initial_timeout=6, + team_infos=None, write_replay=False, store_output=False, exit_flag=None): """ Starts a new process with the given command line arguments and waits until finished. Returns @@ -134,6 +135,8 @@ def call_pelita(team_specs, *, rounds, size, viewer, seed, team_infos=None, writ size = ['--size', size] if size else [] viewer = ['--' + viewer] if viewer else [] seed = ['--seed', seed] if seed else [] + timeout = ['--timeout', str(timeout)] + initial_timeout = ['--initial-timeout', str(initial_timeout)] write_replay = ['--write-replay', write_replay] if write_replay else [] store_output = ['--store-output', store_output] if store_output else [] append_blue = ['--append-blue', team_infos[0]] if team_infos[0] else [] @@ -148,6 +151,8 @@ def call_pelita(team_specs, *, rounds, size, viewer, seed, team_infos=None, writ *size, *viewer, *seed, + *timeout, + *initial_timeout, *write_replay, *store_output] @@ -167,6 +172,11 @@ def call_pelita(team_specs, *, rounds, size, viewer, seed, team_infos=None, writ while True: evts = dict(poll.poll(1000)) + if exit_flag and exit_flag.is_set(): + # An external process tells us to quit + _logger.info("Received exit signal") + break + if not evts and proc.poll() is not None: # no more events and proc has finished. # we break the loop