From fdec9e700f2432ae3b4abb46fdfc912c7f25476e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tim=20Hessenm=C3=BCller?= Date: Mon, 4 May 2020 17:31:20 +0200 Subject: [PATCH] Created python script for analyzing hudson api data --- configuration/hudson.py | 448 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 448 insertions(+) create mode 100644 configuration/hudson.py diff --git a/configuration/hudson.py b/configuration/hudson.py new file mode 100644 index 00000000000..db3637f99c0 --- /dev/null +++ b/configuration/hudson.py @@ -0,0 +1,448 @@ +#!/bin/env python3 +# +# Fetches and analyzes build and test data from the Hudson API +# Run with -h or --help for usage information +# @author Tim Hessenmüller (D062243) +# +try: + import aiohttp +except ImportError: + print("Missing package: aiohttp\nInstall with: pip3 install aiohttp") + exit(1) +import asyncio, logging, random +from typing import List, Tuple, Dict +from argparse import ArgumentParser +from getpass import getpass + +# ---------- Model ---------- + +class WithUrl: + url: str + + def __init__(self, url): + self.url = url + +class NamedWithUrl(WithUrl): + name: str + + def __init__(self, name, url): + super(NamedWithUrl, self).__init__(url) + self.name = name + + +class Case: + name: str + status: str + duration: int + fail_age: int + + def __init__(self, name: str, status: str, duration: int, fail_age: int): + self.name = name + self.status = status + self.duration = duration + self.fail_age = fail_age + +class Suite: + name: str + cases: List[Case] = [] + duration: float + + def __init__(self, name: str, duration: float): + self.name = name + self.duration = duration + +class Build(NamedWithUrl): + suites: List[Suite] = [] + +class Job(NamedWithUrl): + builds: List[Build] = [] + color: str + +class Root(WithUrl): + jobs: List[Job] = [] + +# ---------- Donwloader --------- + +class Downloader: + __session: aiohttp.ClientSession + __lock: asyncio.Semaphore + __all_job_urls: Dict[str, str] = None + base_url: str + api_suffix: str + + logger = logging.getLogger(__name__) + + n_builds = 0 + p_builds = 0 + + def __init__(self, base_url: str, auth: Tuple[str, str], max_concurrent_connections, api_suffix="api/json"): + self.base_url = base_url + self.__session = aiohttp.ClientSession(auth=aiohttp.BasicAuth(auth[0], auth[1])) + self.__lock = asyncio.Semaphore(max_concurrent_connections) + self.api_suffix = api_suffix + + async def close(self): + if self.__session != None: + await self.__session.close() + + async def build_tree(self, jobs: List[str], build_limit: int = 0) -> Root: + if self.__all_job_urls == None: + # Very first request; Test connection and get all existing jobs to compare against + try: + await self.__get_job_list() + except aiohttp.ClientResponseError as e: + self.logger.critical("Connection to Hudson failed:\n" + e) + exit(78) + + root = Root(self.base_url) + for job in jobs: + if job == "*": + root.jobs = [Job(k, v) for k, v in self.__all_job_urls.items()] + break + elif job in self.__all_job_urls: + root.jobs.append(Job(job, self.__all_job_urls.get(job))) + else: + self.logger.warning("Could not find job: " + job) + + await asyncio.gather(*[self.__populate_builds_in_job(job, build_limit) for job in root.jobs]) + return root + + async def __get_job_list(self) -> None: + self.__all_job_urls = {} + json = await self.get_json(self.base_url) + for job in json["jobs"]: + self.__all_job_urls[job["name"]] = job["url"] + + async def __populate_builds_in_job(self, job: Job, build_limit: int) -> None: + try: + json = await self.get_json(job.url) + except aiohttp.ClientResponseError: + return + job.color = json.get("color") + if build_limit <= 0: + for build in json["builds"]: + job.builds.append(Build(str(build["number"]), build["url"])) + else: + for build in json["builds"][0:build_limit]: + job.builds.append(Build(str(build["number"]), build["url"])) + self.n_builds += len(job.builds) + await asyncio.gather(*[self.__populate_suites_in_build(build) for build in job.builds]) + + async def __populate_suites_in_build(self, build: Build) -> None: + try: + json = await self.get_json(build.url + "testReport/api/json", append_api_suffix=False) + except aiohttp.ClientResponseError: + return + build.suites = await asyncio.gather(*[self.__parse_suite_with_cases(suite) for suite in json["suites"]]) + self.p_builds += 1 + + async def __parse_suite_with_cases(self, json: str) -> Suite: + suite = Suite(json.get("name"), json.get("duration")) + suite.cases = [] # For some reason new Suite instances all share the same list + for case_json in json["cases"]: + suite.cases.append(Case(case_json["className"], case_json["status"], case_json["duration"], case_json["age"])) + return suite + + async def get_json(self, url: str, append_api_suffix=True): + if append_api_suffix == True: + if not url.endswith("/"): + url += "/" + url += self.api_suffix + async with self.__lock: + for _ in range(5): + try: + async with self.__session.request("GET", url) as resp: + try: + resp.raise_for_status() + except aiohttp.ClientResponseError as e: + self.logger.debug(e) + raise e + return await resp.json(content_type=None) # Disables content type check + except asyncio.TimeoutError as e: + self.logger.warning(e) + await asyncio.sleep(random.uniform(1, 10)) + self.logger.error(f"5 consecutive timeouts for URL: {url}") + raise aiohttp.ClientResponseError + + def get_progress(self): + if self.n_builds == 0: + return 0 + return self.p_builds / self.n_builds + +# ---------- Analysis ---------- + +class AbstractModule: + def process(self, root: Root): + raise NotImplementedError + def progress(self): + raise NotImplementedError + def result(self) -> str: + return None + +class Analyzer: + __tasks = [] + modules: List[AbstractModule] + + def __init__(self, *modules: AbstractModule): + self.modules = list(modules) + + async def analyze_tree(self, root: Root): + loop = asyncio.get_event_loop() + await asyncio.gather(*[loop.run_in_executor(None, m.process, root) for m in self.modules]) + + def get_progress(self): + if len(self.modules) == 0: + return 0 + acc = 0 + for m in self.modules: + acc += m.progress() + return acc / len(self.modules) + + def print_results(self): + for m in self.modules: + r = m.result() + if r != None: + print(f"Results from {m.__class__.__name__}:\n{r}") + + def add_module(self, module: AbstractModule): + self.modules.append(module) + + def remove_module(self, module: AbstractModule): + self.modules.remove(module) + + +class ModelVisualizerModule(AbstractModule): + __root: Root + level: int + def __init__(self, level: int = 1): + super(ModelVisualizerModule, self).__init__() + self.level = level + def process(self, root: Root): + self.__root = root + def progress(self): + return 1 + def result(self): + out = "" + for job in self.__root.jobs: + out += job.name + ": " + job.color + ", " + str(len(job.builds)) + " builds\n" + if self.level >= 1: + for build in job.builds: + out += " " + build.name + ": " + str(len(build.suites)) + " test suites\n" + if self.level >= 2: + for suite in build.suites: + out += " " + suite.name + ": " + str(len(suite.cases)) + " test cases\n" + return out[:-1] + +class FlakyCasesModule(AbstractModule): + threshold_div: int + case_counter: Dict[str, int] = {} + n_builds = 0 + p_builds = 0 + + def __init__(self, threshold_div: int = 4): + super(FlakyCasesModule, self).__init__() + self.threshold_div = threshold_div + + def process(self, root: Root): + for job in root.jobs: + self.n_builds += len(job.builds) + + for job in root.jobs: + for build in job.builds: + for suite in build.suites: + for case in suite.cases: + if case.fail_age == 1: + self.case_counter[case.name] = self.case_counter.get(case.name, 0) + 1 + self.p_builds += 1 + + def progress(self): + if self.n_builds == 0: + return 0 + return self.p_builds / self.n_builds + + def result(self): + out = "" + s = [(k, self.case_counter.get(k)) for k in sorted(self.case_counter, key=self.case_counter.get, reverse=True)] + for case, new_fails in s: + if (new_fails <= self.n_builds // 4): + break + out += f" {new_fails / self.n_builds:.2f}, {case}\n" + return out[:-1] + +class FailedCasesModule(AbstractModule): + case_counter: Dict[str, int] = {} + n_builds = 0 + p_builds = 0 + + def __init__(self): + super(FailedCasesModule, self).__init__() + + def process(self, root: Root): + for job in root.jobs: + self.n_builds += len(job.builds) + + for job in root.jobs: + for build in job.builds: + for suite in build.suites: + for case in suite.cases: + if case.status == "FAILED" or case.status == "REGRESSION": + self.case_counter[case.name] = self.case_counter.get(case.name, 0) + 1 + self.p_builds += 1 + + def progress(self): + if self.n_builds == 0: + return 0 + return self.p_builds / self.n_builds + + def result(self): + out = "" + s = [(k, self.case_counter.get(k)) for k in sorted(self.case_counter, key=self.case_counter.get, reverse=True)] + for case, fails in s: + out += f"{fails:>5}, {case}\n" + return out[:-1] + +class FailedSuitesModule(FailedCasesModule): + def __init__(self): + super(FailedSuitesModule, self).__init__() + + def process(self, root: Root): + for job in root.jobs: + self.n_builds += len(job.builds) + + for job in root.jobs: + for build in job.builds: + for suite in build.suites: + for case in suite.cases: + if case.status == "FAILED": + self.case_counter[suite.name] = self.case_counter.get(suite.name, 0) + 1 + break + self.p_builds += 1 + +class CaseDurationModule(AbstractModule): + dur_counter = {} + run_counter = {} + n_builds = 0 + p_builds = 0 + def __init__(self): + super(CaseDurationModule, self).__init__() + + def process(self, root: Root): + for job in root.jobs: + self.n_builds += len(job.builds) + + for job in root.jobs: + for build in job.builds: + for suite in build.suites: + for case in suite.cases: + if case.status == "PASSED": + self.dur_counter[case.name] = self.dur_counter.get(case.name, 0) + case.duration + self.run_counter[case.name] = self.run_counter.get(case.name, 0) + 1 + self.p_builds += 1 + + def progress(self): + if self.n_builds == 0: + return 0 + return self.p_builds / self.n_builds + + def result(self): + out = "" + s = [(case, total / self.run_counter.get(case), total) for case, total in self.dur_counter.items()] + s = sorted(s, key=lambda x: x[1], reverse=True) + for case, duration, total in s: + out += f"{duration:>3}s/r ({total:>6}s total): {case}\n" + return out[:-1] + +# ---------- Functions And Helpers ---------- + +class ProgressBar: + __task = None + enabled = True + running = False + __spin = -1 + def start(self, progress_function): + if self.__task == None: + self.__task = asyncio.create_task(self.show(progress_function)) + async def show(self, progress_function): + if not self.enabled: + return + self.running = True + while self.running: + p = progress_function() + print(f"{str(round(p * 100)):>3}% [{self.__bar(p)}] {self.__spinner()}", end="\r", flush=True) + if p >= 1: + self.running = False + else: + await asyncio.sleep(0.5) + print(f"100% [{self.__bar(1)}] ", flush=True) + def __bar(self, p): + width = 20 + pos = round(p * width) + if pos > 0: + out = f"{'=' * (pos - 1)}>" + else: + out = "" + return out + "-" * (width - len(out)) + def __spinner(self): + seq = "-\\|/" + self.__spin = (self.__spin + 1) % len(seq) + return seq[self.__spin] + async def done(self): + self.running = False + if self.__task != None: + await self.__task + self.__task = None + +def parse_args(): + parser = ArgumentParser(description="Fetches and analyzes build and test data from the Hudson API") + parser.add_argument("username", help="Hudson account username") + parser.add_argument("job", nargs="+", help="one or more jobs to fetch") + parser.add_argument("--host", default="https://hudson.sapsailing.com/", help="hostname / IP address of Hudson instance") + parser.add_argument("-c", type=int, default=25, metavar="NUM", help="maximum concurrent connections") + parser.add_argument("-n", type=int, default=0, metavar="LIMIT", help="limits the number of builds per job") + v_group = parser.add_mutually_exclusive_group() + v_group.add_argument("-v", action="store_true") + v_group.add_argument("-q", action="store_true") + args = parser.parse_args() + if not args.host.endswith("/"): + args.host += "/" + return args + +def create_logger(args): + level = logging.INFO + if args.q: + level = logging.ERROR + elif args.v: + level = logging.DEBUG + log_handler = logging.StreamHandler() + log_handler.setFormatter(logging.Formatter("%(asctime)s [%(levelname)s] %(message)s", datefmt="%Y-%m-%dT%H:%M:%S%z")) + log_handler.setLevel(level) + logger = logging.getLogger(__name__) + logger.addHandler(log_handler) + logger.setLevel(level) + return logger + +def get_credentials(username): + password = getpass(f"Enter password: ") + return (username, password) + +async def main(args): + logger = create_logger(args) + + pb = ProgressBar() + pb.enabled = not args.q + d = Downloader(args.host, get_credentials(args.username), args.c) + logger.info("Fetching data...") + pb.start(d.get_progress) + tree = await d.build_tree(args.job, args.n) + await d.close() + await pb.done() + logger.info("Processing data...") + a = Analyzer(ModelVisualizerModule(level=1), FailedCasesModule(), FlakyCasesModule(threshold_div=4)) + pb.start(a.get_progress) + await a.analyze_tree(tree) + await pb.done() + logger.info("Outputting results...") + a.print_results() + +if __name__ == "__main__": + asyncio.run(main(parse_args()))