263 lines
6.9 KiB
Go
263 lines
6.9 KiB
Go
// Copyright 2019 The SwiftShader Authors. All Rights Reserved.
|
|
//
|
|
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
// you may not use this file except in compliance with the License.
|
|
// You may obtain a copy of the License at
|
|
//
|
|
// http://www.apache.org/licenses/LICENSE-2.0
|
|
//
|
|
// Unless required by applicable law or agreed to in writing, software
|
|
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
// See the License for the specific language governing permissions and
|
|
// limitations under the License.
|
|
|
|
// Package testlist provides utilities for handling test lists.
|
|
package testlist
|
|
|
|
import (
|
|
"bytes"
|
|
"crypto/sha1"
|
|
"encoding/gob"
|
|
"encoding/hex"
|
|
"encoding/json"
|
|
"io/ioutil"
|
|
"path/filepath"
|
|
"sort"
|
|
"strings"
|
|
|
|
"../cause"
|
|
)
|
|
|
|
// API is an enumerator of graphics APIs.
|
|
type API string
|
|
|
|
// Graphics APIs.
|
|
const (
|
|
EGL = API("egl")
|
|
GLES2 = API("gles2")
|
|
GLES3 = API("gles3")
|
|
Vulkan = API("vulkan")
|
|
)
|
|
|
|
// Group is a list of tests to be run for a single API.
|
|
type Group struct {
|
|
Name string
|
|
File string
|
|
API API
|
|
Tests []string
|
|
}
|
|
|
|
// Load loads the test list file and appends all tests to the Group.
|
|
func (g *Group) Load() error {
|
|
return g.LoadFile(g.File)
|
|
}
|
|
|
|
func (g *Group) LoadFile(file string) error {
|
|
dir, _ := filepath.Split(file)
|
|
tests, err := ioutil.ReadFile(file)
|
|
if err != nil {
|
|
return cause.Wrap(err, "Couldn't read '%s'", file)
|
|
}
|
|
for _, line := range strings.Split(string(tests), "\n") {
|
|
line = strings.TrimSpace(line)
|
|
// The test list file can contain references to other .txt files
|
|
// containing the individual tests.
|
|
if strings.HasSuffix(line, ".txt") {
|
|
g.LoadFile(filepath.Join(dir, line))
|
|
} else if line != "" && !strings.HasPrefix(line, "#") {
|
|
g.Tests = append(g.Tests, line)
|
|
}
|
|
}
|
|
sort.Strings(g.Tests)
|
|
return nil
|
|
}
|
|
|
|
// Filter returns a new Group that contains only tests that match the predicate.
|
|
func (g Group) Filter(pred func(string) bool) Group {
|
|
out := Group{
|
|
Name: g.Name,
|
|
File: g.File,
|
|
API: g.API,
|
|
}
|
|
for _, test := range g.Tests {
|
|
if pred(test) {
|
|
out.Tests = append(out.Tests, test)
|
|
}
|
|
}
|
|
return out
|
|
}
|
|
|
|
// Limit returns a new Group that contains a maximum of limit tests.
|
|
func (g Group) Limit(limit int) Group {
|
|
out := Group{
|
|
Name: g.Name,
|
|
File: g.File,
|
|
API: g.API,
|
|
Tests: g.Tests,
|
|
}
|
|
if len(g.Tests) > limit {
|
|
out.Tests = g.Tests[:limit]
|
|
}
|
|
return out
|
|
}
|
|
|
|
// Lists is the full list of tests to be run.
|
|
type Lists []Group
|
|
|
|
// Filter returns a new Lists that contains only tests that match the predicate.
|
|
func (l Lists) Filter(pred func(string) bool) Lists {
|
|
out := Lists{}
|
|
for _, group := range l {
|
|
filtered := group.Filter(pred)
|
|
if len(filtered.Tests) > 0 {
|
|
out = append(out, filtered)
|
|
}
|
|
}
|
|
return out
|
|
}
|
|
|
|
// Hash returns a SHA1 hash of the set of tests.
|
|
func (l Lists) Hash() string {
|
|
h := sha1.New()
|
|
if err := gob.NewEncoder(h).Encode(l); err != nil {
|
|
panic(cause.Wrap(err, "Could not encode testlist to produce hash"))
|
|
}
|
|
return hex.EncodeToString(h.Sum(nil))
|
|
}
|
|
|
|
// Load loads the test list json file and returns the full set of tests.
|
|
func Load(root, jsonPath string) (Lists, error) {
|
|
root, err := filepath.Abs(root)
|
|
if err != nil {
|
|
return nil, cause.Wrap(err, "Couldn't get absolute path of '%s'", root)
|
|
}
|
|
|
|
jsonPath, err = filepath.Abs(jsonPath)
|
|
if err != nil {
|
|
return nil, cause.Wrap(err, "Couldn't get absolute path of '%s'", jsonPath)
|
|
}
|
|
|
|
i, err := ioutil.ReadFile(jsonPath)
|
|
if err != nil {
|
|
return nil, cause.Wrap(err, "Couldn't read test list from '%s'", jsonPath)
|
|
}
|
|
|
|
var jsonGroups []struct {
|
|
Name string
|
|
API string
|
|
TestFile string `json:"tests"`
|
|
}
|
|
if err := json.NewDecoder(bytes.NewReader(i)).Decode(&jsonGroups); err != nil {
|
|
return nil, cause.Wrap(err, "Couldn't parse '%s'", jsonPath)
|
|
}
|
|
|
|
dir := filepath.Dir(jsonPath)
|
|
|
|
out := make(Lists, len(jsonGroups))
|
|
for i, jsonGroup := range jsonGroups {
|
|
group := Group{
|
|
Name: jsonGroup.Name,
|
|
File: filepath.Join(dir, jsonGroup.TestFile),
|
|
API: API(jsonGroup.API),
|
|
}
|
|
if err := group.Load(); err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
// Make the path relative before displaying it to the world.
|
|
relPath, err := filepath.Rel(root, group.File)
|
|
if err != nil {
|
|
return nil, cause.Wrap(err, "Couldn't get relative path for '%s'", group.File)
|
|
}
|
|
group.File = relPath
|
|
|
|
out[i] = group
|
|
}
|
|
|
|
return out, nil
|
|
}
|
|
|
|
// Status is an enumerator of test results.
|
|
type Status string
|
|
|
|
const (
|
|
// Pass is the status of a successful test.
|
|
Pass = Status("PASS")
|
|
// Fail is the status of a failed test.
|
|
Fail = Status("FAIL")
|
|
// Timeout is the status of a test that failed to complete in the alloted
|
|
// time.
|
|
Timeout = Status("TIMEOUT")
|
|
// Crash is the status of a test that crashed.
|
|
Crash = Status("CRASH")
|
|
// Unimplemented is the status of a test that failed with UNIMPLEMENTED().
|
|
Unimplemented = Status("UNIMPLEMENTED")
|
|
// Unsupported is the status of a test that failed with UNSUPPORTED().
|
|
Unsupported = Status("UNSUPPORTED")
|
|
// Unreachable is the status of a test that failed with UNREACHABLE().
|
|
Unreachable = Status("UNREACHABLE")
|
|
// Assert is the status of a test that failed with ASSERT() or ASSERT_MSG().
|
|
Assert = Status("ASSERT")
|
|
// Abort is the status of a test that failed with ABORT().
|
|
Abort = Status("ABORT")
|
|
// NotSupported is the status of a test feature not supported by the driver.
|
|
NotSupported = Status("NOT_SUPPORTED")
|
|
// CompatibilityWarning is the status passing test with a warning.
|
|
CompatibilityWarning = Status("COMPATIBILITY_WARNING")
|
|
// QualityWarning is the status passing test with a warning.
|
|
QualityWarning = Status("QUALITY_WARNING")
|
|
// InternalError is the status of a test that failed on an API usage error.
|
|
InternalError = Status("INTERNAL_ERROR")
|
|
)
|
|
|
|
// Statuses is the full list of status types
|
|
var Statuses = []Status{
|
|
Pass,
|
|
Fail,
|
|
Timeout,
|
|
Crash,
|
|
Unimplemented,
|
|
Unsupported,
|
|
Unreachable,
|
|
Assert,
|
|
Abort,
|
|
NotSupported,
|
|
CompatibilityWarning,
|
|
QualityWarning,
|
|
InternalError,
|
|
}
|
|
|
|
// Failing returns true if the task status requires fixing.
|
|
func (s Status) Failing() bool {
|
|
switch s {
|
|
case Fail, Timeout, Crash, Unimplemented, Unreachable, Assert, Abort, InternalError:
|
|
return true
|
|
case Unsupported:
|
|
// This may seem surprising that this should be a failure, however these
|
|
// should not be reached, as dEQP should not be using features that are
|
|
// not advertised.
|
|
return true
|
|
default:
|
|
return false
|
|
}
|
|
}
|
|
|
|
// Passing returns true if the task status is considered a pass.
|
|
func (s Status) Passing() bool {
|
|
switch s {
|
|
case Pass, CompatibilityWarning, QualityWarning:
|
|
return true
|
|
default:
|
|
return false
|
|
}
|
|
}
|
|
|
|
// FilePathWithStatus returns the path to the test list file with the status
|
|
// appended before the file extension.
|
|
func FilePathWithStatus(listPath string, status Status) string {
|
|
ext := filepath.Ext(listPath)
|
|
name := listPath[:len(listPath)-len(ext)]
|
|
return name + "-" + string(status) + ext
|
|
}
|