built_test_history.py
unknown
python
2 years ago
6.9 kB
11
Indexable
from io import TextIOWrapper
import requests
import time
from typing import TypedDict
# This is an example JSON of one of the aligned one objects.
# From: https://wpt.fyi/api/runs?label=master&label=experimental&max-count=100&aligned
#
# id:
# 5078469266898944
# browser_name:
# chrome
# browser_version:
# 114.0.5696.0 dev
# os_name:
# linux
# os_version:
# 20.04
# revision:
# 0375b3ebd8
# full_revision_hash:
# 0375b3ebd812ae5d8713e3bb06c5805273ebbe3b
# results_url:
# https://storage.googleapis.com/wptd/0375b3ebd812ae5d8713e3bb06c5805273ebbe3b/chrome-114.0.5696.0_dev-linux-20.04-c84aaf98a5-summary_v2.json.gz
# created_at:
# 2023-04-13T04:20:50.316333Z
# time_start:
# 2023-04-13T02:57:17.041Z
# time_end:
# 2023-04-13T04:04:37.36Z
# raw_results_url:
# https://storage.googleapis.com/wptd-results/0375b3ebd812ae5d8713e3bb06c5805273ebbe3b/chrome-114.0.5696.0_dev-linux-20.04-c84aaf98a5/report.json
# labels:
# ['chrome', 'dev', 'experimental', 'master', 'taskcluster', 'user:chromium-wpt-export-bot']
# Type hint class for the run metadata return value from api/runs endpoint.
class MetadataDict(TypedDict):
id: str
browser_name: str
browser_version: str
os_name: str
os_version: str
revision: str
full_revision_hash: str
results_url: str
created_at: str
time_start: str
time_end: str
raw_results_url: str
labels: list[str]
# {
# ('/wasm/jsapi/table/constructor-types.tentative.any.js', ''): 'SKIP',
# ('/wasm/jsapi/table/constructor-types.tentative.any.js', 'Zero minimum'): 'FAIL',
# }
# Type hint class for the dictionary maintaining previous test/subtest statuses.
class PrevTestStatusDict(TypedDict):
chrome: dict[tuple[str, str], str]
edge: dict[tuple[str, str], str]
firefox: dict[tuple[str, str], str]
safari: dict[tuple[str, str], str]
# Get the list of metadata for the most recent aligned runs.
def get_aligned_run_info() -> list[MetadataDict]:
# Change the "max-count" to try this script with a smaller set.
resp = requests.get('https://wpt.fyi/api/runs?label=master&label=experimental&max-count=2&aligned')
runs_list: list[MetadataDict] = resp.json()
# Sort by browser name -> then time start, so that the aligned runs are
# processed in groups with each other.
runs_list.sort(key=lambda run: run['browser_name'])
runs_list.sort(key=lambda run: run['time_start'])
# Print the dates just to get info on the list of runs we're working with.
print('Runs to process:')
for run in runs_list:
print(f'{run["browser_name"]} {run["time_start"]}')
print()
return runs_list
# A little loading bar for fun.
def print_loading_bar(i: int, run_count: int) -> None:
run_number = i + 1
print(f'|{"#" * run_number}{"-" * (run_count - run_number)}| '
f'({run_number}/{run_count})')
def write_row_if_needed(
test_name: str,
subtest_name: str,
key: tuple[str, str],
current_status: str,
prev_test_statuses: dict[str, str],
csv_row_metadata: str,
file: TextIOWrapper,
) -> None:
# If the test status differs from the previous run, write a row.
if current_status != prev_test_statuses.get(key, ''):
file.write('{},"{}","{}",{}\n'.format(
csv_row_metadata,
test_name,
subtest_name,
current_status)
)
# Replace the previous status with this new status.
prev_test_statuses[key] = current_status
def process_single_run(
run_metadata: MetadataDict,
prev_test_statuses: dict[str, str],
) -> None:
with open('browser-history-test-data.csv', 'a') as f:
run_resp = requests.get(run_metadata['raw_results_url'])
run_data = run_resp.json()
# Create the first columns of each row with overall run info.
csv_row_metadata = '{},{},{},{}'.format(
run_metadata['id'],
run_metadata['browser_name'],
run_metadata['browser_version'],
run_data['time_start'],
)
# Keep track of every single test result that's in the dataset of
# runs we've previously seen. If they're not in the run we're processing,
# we'll mark them as missing.
tests_not_seen = set(prev_test_statuses.keys())
# iterate through each test.
for test_data in run_data['results']:
# Format the test name.
test_name = (test_data['test']
.replace('\"', '\"\"').replace('\n', ' '))
# Test results are stored in dictionary with a tuple key
# in the form of (testname, subtest_name).
# The overall test status has an empty string as the subtest name.
test_key = (test_name, '')
write_row_if_needed(
test_name,
subtest_name='',
key=test_key,
current_status=test_data['status'],
prev_test_statuses=prev_test_statuses,
csv_row_metadata=csv_row_metadata,
file=f,
)
# Now that we've seen this test status, we can remove it from the
# the set of tests we haven't seen yet.
tests_not_seen.discard(test_key)
# Do the same basic process for each subtest.
for subtest_data in test_data['subtests']:
subtest_name = (subtest_data['name']
.replace('\"', '\"\"').replace('\n', ' '))
subtest_key = (test_name, subtest_name)
write_row_if_needed(
test_name,
subtest_name=subtest_name,
key=test_key,
current_status=subtest_data['status'],
prev_test_statuses=prev_test_statuses,
csv_row_metadata=csv_row_metadata,
file=f,
)
tests_not_seen.discard(subtest_key)
# Write MISSING status for tests/subtests not seen.
for test_name, subtest_name in tests_not_seen:
key = (test_name, subtest_name)
# Only write a row as missing if it's not already marked as missing.
write_row_if_needed(
test_name,
subtest_name=subtest_name,
key=key,
current_status='MISSING',
prev_test_statuses=prev_test_statuses,
csv_row_metadata=csv_row_metadata,
file=f,
)
def process_runs(runs_list: list[MetadataDict]) -> None:
# Time the process
start = time.time()
# Create first CSV columns
with open('browser-history-test-data.csv', 'w') as f:
f.write('run_id,browser_name,browser_version,date,test_name,subtest_name,status\n')
# Keep a dictionary of the previous test statuses from runs we've processed.
prev_test_statuses: PrevTestStatusDict = {
'chrome': {},
'edge': {},
'firefox': {},
'safari': {},
}
# Go through each aligned run.
for i, run_metadata in enumerate(runs_list):
browser_name = run_metadata['browser_name']
process_single_run(
run_metadata,
prev_test_statuses[browser_name],
)
print_loading_bar(i, len(runs_list))
print(f'Time taken = {round(time.time() - start, 0)} seconds.')
def main():
runs_list = get_aligned_run_info()
process_runs(runs_list)
if __name__ == '__main__':
main()
Editor is loading...