# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
import errno
import json
import os
import sys
from pathlib
import Path
from mozfile
import which
from mozperftest.layers
import Layer
from mozperftest.utils
import run_script, silence
METRICS_FIELDS = (
"SpeedIndex",
"FirstVisualChange",
"LastVisualChange",
"VisualProgress",
"videoRecordingStart",
)
class VisualData:
def open_data(self, data):
res = {
"name":
"visualmetrics",
"subtest": data[
"name"],
"data": [
{
"file":
"visualmetrics",
"value": value,
"xaxis": xaxis}
for xaxis, value
in enumerate(data[
"values"])
],
}
return res
def transform(self, data):
return data
def merge(self, data):
return data
class VisualMetrics(Layer):
"""Wrapper around Browsertime's visualmetrics.py script"""
name =
"visualmetrics"
activated =
False
arguments = {}
def setup(self):
self.metrics = {}
self.metrics_fields = []
# making sure we have ffmpeg and imagemagick available
for tool
in (
"ffmpeg",
"convert"):
if sys.platform
in (
"win32",
"msys"):
tool +=
".exe"
path = which(tool)
if not path:
raise OSError(errno.ENOENT, f
"Could not find {tool}")
def run(self, metadata):
if "VISUALMETRICS_PY" not in os.environ:
raise OSError(
"The VISUALMETRICS_PY environment variable is not set."
"Make sure you run the browsertime layer"
)
path = Path(os.environ[
"VISUALMETRICS_PY"])
if not path.exists():
raise FileNotFoundError(str(path))
self.visualmetrics = path
treated = 0
for result
in metadata.get_results():
result_dir = result.get(
"results")
if result_dir
is None:
continue
result_dir = Path(result_dir)
if not result_dir.is_dir():
continue
browsertime_json = Path(result_dir,
"browsertime.json")
if not browsertime_json.exists():
continue
treated += self.run_visual_metrics(browsertime_json)
self.info(f
"Treated {treated} videos.")
if len(self.metrics) > 0:
metadata.add_result(
{
"name": metadata.script[
"name"] +
"-vm",
"framework": {
"name":
"mozperftest"},
"transformer":
"mozperftest.metrics.visualmetrics:VisualData",
"results": list(self.metrics.values()),
}
)
# we also extend --perfherder-metrics and --console-metrics if they
# are activated
def add_to_option(name):
existing = self.get_arg(name, [])
for field
in self.metrics_fields:
existing.append({
"name": field,
"unit":
"ms"})
self.env.set_arg(name, existing)
if self.get_arg(
"perfherder"):
add_to_option(
"perfherder-metrics")
if self.get_arg(
"console"):
add_to_option(
"console-metrics")
else:
self.warning(
"No video was treated.")
return metadata
def run_visual_metrics(self, browsertime_json):
verbose = self.get_arg(
"verbose")
self.info(f
"Looking at {browsertime_json}")
venv = self.mach_cmd.virtualenv_manager
class _display:
def __enter__(self, *args, **kw):
return self
__exit__ = __enter__
may_silence =
not verbose
and silence
or _display
with browsertime_json.open()
as f:
browsertime_json_data = json.loads(f.read())
videos = 0
global_options = [
str(self.visualmetrics),
"--orange",
"--perceptual",
"--contentful",
"--force",
"--renderignore",
"5",
"--viewport",
]
if verbose:
global_options += [
"-vvv"]
for site
in browsertime_json_data:
# collecting metrics from browserScripts
# because it can be used in splitting
for index, bs
in enumerate(site[
"browserScripts"]):
for name, val
in bs.items():
if not isinstance(val, (str, int)):
continue
self.append_metrics(index, name, val)
extra = {
"lowerIsBetter":
True,
"unit":
"ms"}
for index, video
in enumerate(site[
"files"][
"video"]):
videos += 1
video_path = browsertime_json.parent / video
output =
"[]"
with may_silence():
res, output = run_script(
venv.python_path,
global_options + [
"--video", str(video_path),
"--json"],
verbose=verbose,
label=
"visual metrics",
display=
False,
)
if not res:
self.error(f
"Failed {res}")
continue
output = output.strip()
if verbose:
self.info(str(output))
try:
output = json.loads(output)
except json.JSONDecodeError:
self.error(
"Could not read the json output from visualmetrics.py")
continue
for name, value
in output.items():
if name.endswith(
"Progress",
):
self._expand_visual_progress(index, name, value, **extra)
else:
self.append_metrics(index, name, value, **extra)
return videos
def _expand_visual_progress(self, index, name, value, **fields):
def _split_percent(val):
# value is of the form "567=94%"
val = val.split(
"=")
value, percent = val[0].strip(), val[1].strip()
if percent.endswith(
"%"):
percent = percent[:-1]
return int(percent), int(value)
percents = [_split_percent(elmt)
for elmt
in value.split(
",")]
# we want to keep the first added value for each percent
# so the trick here is to create a dict() with the reversed list
percents = dict(reversed(percents))
# we are keeping the last 5 percents
percents = list(percents.items())
percents.sort()
for percent, value
in percents[:5]:
self.append_metrics(index, f
"{name}{percent}", value, **fields)
def append_metrics(self, index, name, value, **fields):
if name
not in self.metrics_fields:
self.metrics_fields.append(name)
if name
not in self.metrics:
self.metrics[name] = {
"name": name,
"values": []}
self.metrics[name][
"values"].append(value)
self.metrics[name].update(**fields)
