Introduce end2end tests.

This commit adds a test harness that handles instantiating tests on
multiple backends, and have deferred expectations on the content of
resources.

src/tests/CMakeLists.txt

@@ -15,6 +15,8 @@
 set(TESTS_DIR ${CMAKE_CURRENT_SOURCE_DIR})
 set(UNITTESTS_DIR ${TESTS_DIR}/unittests)
 set(VALIDATION_TESTS_DIR ${UNITTESTS_DIR}/validation)
+set(END2END_TESTS_DIR ${TESTS_DIR}/end2end)
+
 add_executable(nxt_unittests
     ${UNITTESTS_DIR}/BitSetIteratorTests.cpp
     ${UNITTESTS_DIR}/CommandAllocatorTests.cpp
@@ -40,3 +42,13 @@ add_executable(nxt_unittests
 target_link_libraries(nxt_unittests gtest nxt_backend mock_nxt nxt_wire)
 target_include_directories(nxt_unittests PRIVATE ${SRC_DIR})
 SetCXX14(nxt_unittests)
+
+add_executable(nxt_end2end_tests
+    ${END2END_TESTS_DIR}/BasicTests.cpp
+    ${TESTS_DIR}/End2EndTestsMain.cpp
+    ${TESTS_DIR}/NXTTest.cpp
+    ${TESTS_DIR}/NXTTest.h
+)
+target_link_libraries(nxt_end2end_tests gtest utils)
+target_include_directories(nxt_end2end_tests PRIVATE ${SRC_DIR})
+SetCXX14(nxt_end2end_tests)

src/tests/End2EndTestsMain.cpp

@@ -0,0 +1,20 @@
+// Copyright 2017 The NXT Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <gtest/gtest.h>
+
+int main(int argc, char** argv) {
+    testing::InitGoogleTest(&argc, argv);
+    return RUN_ALL_TESTS();
+}

src/tests/NXTTest.cpp

@@ -0,0 +1,275 @@
+// Copyright 2017 The NXT Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "NXTTest.h"
+
+#include "utils/BackendBinding.h"
+
+#include "GLFW/glfw3.h"
+
+#include <cassert>
+#define ASSERT assert
+
+namespace {
+
+    utils::BackendType ParamToBackendType(BackendType type) {
+        switch(type) {
+            case D3D12Backend:
+                return utils::BackendType::D3D12;
+            case MetalBackend:
+                return utils::BackendType::Metal;
+            case OpenGLBackend:
+                return utils::BackendType::OpenGL;
+            case VulkanBackend:
+                return utils::BackendType::Vulkan;
+            default:
+                ASSERT(false);
+                return utils::BackendType::Null;
+        }
+    }
+
+    std::string ParamName(BackendType type) {
+        switch(type) {
+            case D3D12Backend:
+                return "D3D12";
+            case MetalBackend:
+                return "Metal";
+            case OpenGLBackend:
+                return "OpenGL";
+            case VulkanBackend:
+                return "Vulkan";
+            default:
+                ASSERT(false);
+                return "";
+        }
+    }
+
+    // Windows don't usually like to be bound to one API than the other, for example switching
+    // from Vulkan to OpenGL causes crashes on some drivers. Because of this, we lazily created
+    // a window for each backing API.
+    GLFWwindow* windows[NumBackendTypes];
+
+    // Creates a GLFW window set up for use with a given backend.
+    GLFWwindow* GetWindowForBackend(utils::BackendBinding* binding, BackendType type) {
+        GLFWwindow** window = &windows[type];
+
+        if (*window != nullptr) {
+            return *window;
+        }
+
+        if (!glfwInit()) {
+            return nullptr;
+        }
+
+        glfwDefaultWindowHints();
+        binding->SetupGLFWWindowHints();
+
+        std::string windowName = "NXT " + ParamName(type) + " test window";
+        *window = glfwCreateWindow(400, 400, windowName.c_str(), nullptr, nullptr);
+
+        return *window;
+    }
+
+    // End2end tests should test valid commands produce the expected result so no error
+    // should happen. Failure cases should be tested in the validation tests.
+    void DeviceErrorCauseTestFailure(const char* message, nxtCallbackUserdata) {
+         FAIL() << "Device level failure: " << message;
+    }
+
+    struct MapReadUserdata {
+        NXTTest* test;
+        size_t slot;
+    };
+}
+
+NXTTest::~NXTTest() {
+    // We need to destroy child objects before the Device
+    readbackSlots.clear();
+    queue = nxt::Queue();
+    device = nxt::Device();
+
+    delete binding;
+    binding = nullptr;
+
+    nxtSetProcs(nullptr);
+}
+
+void NXTTest::SetUp() {
+    binding = utils::CreateBinding(ParamToBackendType(GetParam()));
+    ASSERT(binding != nullptr);
+
+    GLFWwindow* testWindow = GetWindowForBackend(binding, GetParam());
+    ASSERT(testWindow != nullptr);
+
+    binding->SetWindow(testWindow);
+
+    nxtDevice backendDevice;
+    nxtProcTable backendProcs;
+    binding->GetProcAndDevice(&backendProcs, &backendDevice);
+
+    nxtSetProcs(&backendProcs);
+    device = nxt::Device::Acquire(backendDevice);
+    queue = device.CreateQueueBuilder().GetResult();
+
+    device.SetErrorCallback(DeviceErrorCauseTestFailure, 0);
+}
+
+void NXTTest::TearDown() {
+    MapSlotsSynchronously();
+    ResolveExpectations();
+
+    for (auto& expectation : deferredExpectations) {
+        delete expectation.expectation;
+        expectation.expectation = nullptr;
+    }
+}
+
+void NXTTest::AddBufferExpectation(const char* file, int line, const nxt::Buffer& buffer, uint32_t offset, uint32_t size, detail::Expectation* expectation) {
+    nxt::Buffer source = buffer.Clone();
+
+    auto readback = ReserveReadback(size);
+
+    // We need to enqueue the copy immediately because by the time we resolve the expectation,
+    // the buffer might have been modified.
+    nxt::CommandBuffer commands = device.CreateCommandBufferBuilder()
+        .TransitionBufferUsage(source, nxt::BufferUsageBit::TransferSrc)
+        .TransitionBufferUsage(readback.buffer, nxt::BufferUsageBit::TransferDst)
+        .CopyBufferToBuffer(source, offset, readback.buffer, readback.offset, size)
+        .GetResult();
+
+    queue.Submit(1, &commands);
+
+    DeferredExpectation deferred;
+    deferred.file = file;
+    deferred.line = line;
+    deferred.readbackSlot = readback.slot;
+    deferred.readbackOffset = readback.offset;
+    deferred.size = size;
+    deferred.expectation = expectation;
+
+    deferredExpectations.push_back(deferred);
+}
+
+NXTTest::ReadbackReservation NXTTest::ReserveReadback(uint32_t readbackSize) {
+    // For now create a new MapRead buffer for each readback
+    // TODO(cwallez@chromium.org): eventually make bigger buffers and allocate linearly?
+    ReadbackSlot slot;
+    slot.bufferSize = readbackSize;
+    slot.buffer = device.CreateBufferBuilder()
+        .SetSize(readbackSize)
+        .SetAllowedUsage(nxt::BufferUsageBit::MapRead | nxt::BufferUsageBit::TransferDst)
+        .SetInitialUsage(nxt::BufferUsageBit::TransferDst)
+        .GetResult();
+
+    ReadbackReservation reservation;
+    reservation.buffer = slot.buffer.Clone();
+    reservation.slot = readbackSlots.size();
+    reservation.offset = 0;
+
+    readbackSlots.push_back(std::move(slot));
+    return reservation;
+}
+
+void NXTTest::MapSlotsSynchronously() {
+    // Initialize numPendingMapOperations before mapping, just in case the callback is called immediately.
+    numPendingMapOperations = readbackSlots.size();
+
+    // Map all readback slots
+    for (size_t i = 0; i < readbackSlots.size(); ++i) {
+        auto userdata = new MapReadUserdata{this, i};
+
+        auto& slot = readbackSlots[i];
+        slot.buffer.TransitionUsage(nxt::BufferUsageBit::MapRead);
+        slot.buffer.MapReadAsync(0, slot.bufferSize, SlotMapReadCallback, static_cast<nxt::CallbackUserdata>(reinterpret_cast<uintptr_t>(userdata)));
+    }
+
+    // Busy wait until all map operations are done.
+    // TODO(cwallez@chromium.org): usleep a bit?
+    while (numPendingMapOperations != 0) {
+        device.Tick();
+    }
+}
+
+// static
+void NXTTest::SlotMapReadCallback(nxtBufferMapReadStatus status, const void* data, nxtCallbackUserdata userdata_) {
+    ASSERT(status == NXT_BUFFER_MAP_READ_STATUS_SUCCESS);
+
+    auto userdata = reinterpret_cast<MapReadUserdata*>(static_cast<uintptr_t>(userdata_));
+    userdata->test->readbackSlots[userdata->slot].mappedData = data;
+    userdata->test->numPendingMapOperations --;
+
+    delete userdata;
+}
+
+void NXTTest::ResolveExpectations() {
+    for(const auto& expectation : deferredExpectations) {
+        ASSERT(readbackSlots[expectation.readbackSlot].mappedData != nullptr);
+
+        // Get a pointer to the mapped copy of the data for the expectation.
+        const char* data = reinterpret_cast<const char*>(readbackSlots[expectation.readbackSlot].mappedData);
+        data += expectation.readbackOffset;
+
+        // Get the result for the expectation and add context to failures
+        testing::AssertionResult result = expectation.expectation->Check(data, expectation.size);
+        if (!result) {
+            result << " Expectation created at " << expectation.file << ":" << expectation.line;
+        }
+
+        EXPECT_TRUE(result);
+    }
+}
+
+namespace detail {
+    bool IsBackendAvailable(BackendType type) {
+        #if defined(__APPLE__)
+            return type == MetalBackend;
+        #else
+            return false;
+        #endif
+    }
+
+    std::vector<BackendType> FilterBackends(const BackendType* types, size_t numParams) {
+        std::vector<BackendType> backends;
+
+        for (size_t i = 0; i < numParams; ++i) {
+            if (IsBackendAvailable(types[i])) {
+                backends.push_back(types[i]);
+            }
+        }
+        return backends;
+    }
+
+    // Helper classes to set expectations
+
+    template<typename T>
+    ExpectEq<T>::ExpectEq(T singleValue) {
+        expected.push_back(singleValue);
+    }
+
+    template<typename T>
+    testing::AssertionResult ExpectEq<T>::Check(const void* data, size_t size) {
+        ASSERT(size == sizeof(T) * expected.size());
+
+        const T* actual = reinterpret_cast<const T*>(data);
+        for (size_t i = 0; i < expected.size(); ++i) {
+            if (actual[i] != expected[i]) {
+                return testing::AssertionFailure() << "Expected data[" << i << "] to be " << expected[i] << ", actual " << actual[i] << std::endl;
+            }
+        }
+
+        return testing::AssertionSuccess();
+    }
+
+    template class ExpectEq<uint32_t>;
+}

src/tests/NXTTest.h

@@ -0,0 +1,129 @@
+// Copyright 2017 The NXT Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "nxt/nxtcpp.h"
+
+#include <gtest/gtest.h>
+
+// Getting data back from NXT is done in an async manners so all expectations are "deferred"
+// until the end of the test. Also expectations use a copy to a MapRead buffer to get the data
+// so resources should have the TransferDst allowed usage bit if you want to add expectations on them.
+#define EXPECT_BUFFER_U32_EQ(expected, buffer, offset) \
+    AddBufferExpectation(__FILE__, __LINE__, buffer, offset, sizeof(uint32_t), new detail::ExpectEq<uint32_t>(expected));
+
+// Backend types used in the NXT_INSTANTIATE_TEST
+enum BackendType {
+    D3D12Backend,
+    MetalBackend,
+    OpenGLBackend,
+    VulkanBackend,
+    NumBackendTypes,
+};
+
+namespace utils {
+    class BackendBinding;
+}
+
+namespace detail {
+    class Expectation;
+}
+
+class NXTTest : public ::testing::TestWithParam<BackendType> {
+    public:
+        ~NXTTest();
+
+        void SetUp() override;
+        void TearDown() override;
+
+    protected:
+        nxt::Device device;
+        nxt::Queue queue;
+
+        // Helper methods to implement the EXPECT_ macros
+        void AddBufferExpectation(const char* file, int line, const nxt::Buffer& buffer, uint32_t offset, uint32_t size, detail::Expectation* expectation);
+
+    private:
+        // MapRead buffers used to get data for the expectations
+        struct ReadbackSlot {
+            nxt::Buffer buffer;
+            uint32_t bufferSize;
+            const void* mappedData = nullptr;
+        };
+        std::vector<ReadbackSlot> readbackSlots;
+
+        // Maps all the buffers and fill ReadbackSlot::mappedData
+        void MapSlotsSynchronously();
+        static void SlotMapReadCallback(nxtBufferMapReadStatus status, const void* data, nxtCallbackUserdata userdata);
+        size_t numPendingMapOperations = 0;
+
+        // Reserve space where the data for an expectation can be copied
+        struct ReadbackReservation {
+            nxt::Buffer buffer;
+            size_t slot;
+            uint32_t offset;
+        };
+        ReadbackReservation ReserveReadback(uint32_t readbackSize);
+
+        struct DeferredExpectation {
+            const char* file;
+            int line;
+            size_t readbackSlot;
+            uint32_t readbackOffset;
+            uint32_t size;
+            detail::Expectation* expectation;
+        };
+        std::vector<DeferredExpectation> deferredExpectations;
+
+        // Assuming the data is mapped, checks all expectations
+        void ResolveExpectations();
+
+        utils::BackendBinding* binding = nullptr;
+};
+
+// Instantiate the test once for each backend provided after the first argument. Use it like this:
+//     NXT_INSTANTIATE_TEST(MyTestFixture, OpenGLBackend, MetalBackend)
+#define NXT_INSTANTIATE_TEST(testName, firstParam, ...) \
+    const decltype(firstParam) testName##params[] = { firstParam, ##__VA_ARGS__ }; \
+    INSTANTIATE_TEST_CASE_P(, testName, \
+        testing::ValuesIn(::detail::FilterBackends(testName##params, sizeof(testName##params) / sizeof(firstParam))), \
+        testing::PrintToStringParamName());
+
+namespace detail {
+    // Helper functions used for NXT_INSTANTIATE_TEST
+    bool IsBackendAvailable(BackendType type);
+    std::vector<BackendType> FilterBackends(const BackendType* types, size_t numParams);
+
+    // All classes used to implement the deferred expectations should inherit from this.
+    class Expectation {
+        public:
+            virtual ~Expectation() = default;
+
+            // Will be called with the buffer or texture data the expectation should check.
+            virtual testing::AssertionResult Check(const void* data, size_t size) = 0;
+    };
+
+    // Expectation that checks the data is equal to some expected values.
+    template<typename T>
+    class ExpectEq : public Expectation {
+        public:
+            ExpectEq(T singleValue);
+
+            testing::AssertionResult Check(const void* data, size_t size) override;
+
+        private:
+            std::vector<T> expected;
+    };
+    extern template class ExpectEq<uint32_t>;
+}
+

src/tests/end2end/BasicTests.cpp

@@ -0,0 +1,35 @@
+// Copyright 2017 The NXT Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "tests/NXTTest.h"
+
+class BasicTests : public NXTTest {
+};
+
+// Test Buffer::SetSubData changes the content of the buffer, but really this is the most
+// basic test possible, and tests the test harness
+TEST_P(BasicTests, BufferSetSubData) {
+    nxt::Buffer buffer = device.CreateBufferBuilder()
+        .SetSize(4)
+        .SetAllowedUsage(nxt::BufferUsageBit::TransferSrc | nxt::BufferUsageBit::TransferDst)
+        .SetInitialUsage(nxt::BufferUsageBit::TransferDst)
+        .GetResult();
+
+    uint32_t value = 3094587;
+    buffer.SetSubData(0, 1, &value);
+
+    EXPECT_BUFFER_U32_EQ(value, buffer, 0);
+}
+
+NXT_INSTANTIATE_TEST(BasicTests, MetalBackend)

src/utils/BackendBinding.h

@@ -31,6 +31,8 @@ namespace utils {
 
     class BackendBinding {
         public:
+            virtual ~BackendBinding() = default;
+
             virtual void SetupGLFWWindowHints() = 0;
             virtual void GetProcAndDevice(nxtProcTable* procs, nxtDevice* device) = 0;
             virtual void SwapBuffers() = 0;