[tools]: Add remote-compile

A tool for compiling shaders on a remote machine.
Helpful for combining with `test-all`, so that a single non-windows
machine can validate SPIR-V, MSL and HLSL.

Change-Id: I3a0f70e6e4edd13952eb5dc72fbbed7c495036ee
Reviewed-on: https://dawn-review.googlesource.com/c/tint/+/56940
Auto-Submit: Ben Clayton <bclayton@google.com>
Kokoro: Kokoro <noreply+kokoro@google.com>
Reviewed-by: Antonio Maiorano <amaiorano@google.com>
Commit-Queue: Ben Clayton <bclayton@google.com>
This commit is contained in:
Ben Clayton 2021-07-08 19:35:53 +00:00 committed by Tint LUCI CQ
parent d09317c0b5
commit 54cfa0b9a1
10 changed files with 1150 additions and 9 deletions

View File

@ -53,6 +53,7 @@ option(TINT_BUILD_FUZZERS "Build fuzzers" OFF)
option(TINT_BUILD_SPIRV_TOOLS_FUZZER "Build SPIRV-Tools fuzzer" OFF)
option(TINT_BUILD_TESTS "Build tests" ${TINT_BUILD_TESTS_DEFAULT})
option(TINT_BUILD_AS_OTHER_OS "Override OS detection to force building of *_other.cc files" OFF)
option(TINT_BUILD_REMOTE_COMPILE "Build the remote-compile tool for validating shaders on a remote machine" OFF)
option(TINT_ENABLE_MSAN "Enable memory sanitizer" OFF)
option(TINT_ENABLE_ASAN "Enable address sanitizer" OFF)
@ -77,6 +78,7 @@ message(STATUS "Tint build with ASAN: ${TINT_ENABLE_ASAN}")
message(STATUS "Tint build with MSAN: ${TINT_ENABLE_MSAN}")
message(STATUS "Tint build with UBSAN: ${TINT_ENABLE_UBSAN}")
message(STATUS "Tint build checking [chromium-style]: ${TINT_CHECK_CHROMIUM_STYLE}")
message(STATUS "Tint build remote-compile tool: ${TINT_BUILD_REMOTE_COMPILE}")
message(STATUS "Using python3")
find_package(PythonInterp 3 REQUIRED)
@ -99,6 +101,8 @@ if (${TINT_BUILD_SPIRV_TOOLS_FUZZER})
set(TINT_BUILD_SPV_WRITER ON)
endif()
set(TINT_ROOT_SOURCE_DIR ${PROJECT_SOURCE_DIR})
# CMake < 3.15 sets /W3 in CMAKE_CXX_FLAGS. Remove it if it's there.
# See https://gitlab.kitware.com/cmake/cmake/-/issues/18317
if (MSVC)
@ -112,7 +116,7 @@ if (${TINT_CHECK_CHROMIUM_STYLE})
endif()
if (${TINT_BUILD_SPV_READER})
include_directories("${PROJECT_SOURCE_DIR}/third_party/spirv-tools/include")
include_directories("${TINT_ROOT_SOURCE_DIR}/third_party/spirv-tools/include")
endif()
if((CMAKE_CXX_COMPILER_ID STREQUAL "Clang") AND (CMAKE_CXX_SIMULATE_ID STREQUAL "MSVC"))
@ -171,12 +175,12 @@ if(${TINT_BUILD_DOCS})
endif()
function(tint_default_compile_options TARGET)
target_include_directories(${TARGET} PUBLIC "${PROJECT_SOURCE_DIR}")
target_include_directories(${TARGET} PUBLIC "${PROJECT_SOURCE_DIR}/include")
target_include_directories(${TARGET} PUBLIC "${TINT_ROOT_SOURCE_DIR}")
target_include_directories(${TARGET} PUBLIC "${TINT_ROOT_SOURCE_DIR}/include")
if (${TINT_BUILD_SPV_READER} OR ${TINT_BUILD_SPV_WRITER})
target_include_directories(${TARGET} PUBLIC
"${PROJECT_SOURCE_DIR}/third_party/spirv-headers/include")
"${TINT_ROOT_SOURCE_DIR}/third_party/spirv-headers/include")
endif()
target_compile_definitions(${TARGET} PUBLIC
@ -186,7 +190,7 @@ function(tint_default_compile_options TARGET)
target_compile_definitions(${TARGET} PUBLIC
-DTINT_BUILD_HLSL_WRITER=$<BOOL:${TINT_BUILD_HLSL_WRITER}>)
target_compile_definitions(${TARGET} PUBLIC
-DTINT_BUILD_MSL_WRITER=$<BOOL:${TINT_BUILD_MSL_WRITER}>)
-DTINT_BUILD_MSL_WRITER=$<BOOL:${TINT_BUILD_MSL_WRITER}>)
target_compile_definitions(${TARGET} PUBLIC
-DTINT_BUILD_SPV_WRITER=$<BOOL:${TINT_BUILD_SPV_WRITER}>)
target_compile_definitions(${TARGET} PUBLIC
@ -304,13 +308,13 @@ endif()
add_custom_target(tint-lint
COMMAND ./tools/lint
WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}
WORKING_DIRECTORY ${TINT_ROOT_SOURCE_DIR}
COMMENT "Running linter"
VERBATIM)
add_custom_target(tint-format
COMMAND ./tools/format
WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}
WORKING_DIRECTORY ${TINT_ROOT_SOURCE_DIR}
COMMENT "Running formatter"
VERBATIM)
@ -324,7 +328,11 @@ if (${TINT_EMIT_COVERAGE} AND CMAKE_CXX_COMPILER_ID MATCHES "Clang")
add_custom_target(tint-generate-coverage
COMMAND ${CMAKE_COMMAND} -E env PATH=${PATH_WITH_CLANG} ./tools/tint-generate-coverage $<TARGET_FILE:tint_unittests>
DEPENDS tint_unittests
WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}
WORKING_DIRECTORY ${TINT_ROOT_SOURCE_DIR}
COMMENT "Generating tint coverage data"
VERBATIM)
endif()
if (${TINT_BUILD_REMOTE_COMPILE})
add_subdirectory("${TINT_ROOT_SOURCE_DIR}/tools/src/cmd/remote-compile")
endif()

View File

@ -786,8 +786,9 @@ WARN_LOGFILE =
# Note: If this tag is empty the current directory is searched.
INPUT = CODE_OF_CONDUCT.md \
fuzzers/tint_spirv_tools_fuzzer \
src \
fuzzers/tint_spirv_tools_fuzzer
tools/src \
# This tag can be used to specify the character encoding of the source files
# that doxygen parses. Internally doxygen uses the UTF-8 encoding. Doxygen uses

View File

@ -25,6 +25,7 @@ fi
FILTER="-runtime/references"
FILES="`find src -type f` `find samples -type f`"
FILES+="`find tools/src -type f` `find samples -type f`"
if command -v go &> /dev/null; then
# Go is installed. Run cpplint in parallel for speed wins

View File

@ -0,0 +1,37 @@
# Copyright 2021 The Tint Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
set(SRC
main.cc
rwmutex.h
socket.cc
socket.h
)
add_executable(tint-remote-compile ${SRC})
target_include_directories(tint-remote-compile PRIVATE "${TINT_ROOT_SOURCE_DIR}")
# If we're building on mac / ios and we have CoreGraphics, then we can use the
# metal API to validate our shaders. This is roughly 4x faster than invoking
# the metal shader compiler executable.
if(APPLE)
find_library(LIB_CORE_GRAPHICS CoreGraphics)
if(LIB_CORE_GRAPHICS)
target_sources(tint-remote-compile PRIVATE "msl_metal.mm")
target_compile_definitions(tint-remote-compile PRIVATE "-DTINT_ENABLE_MSL_COMPILATION_USING_METAL_API=1")
target_compile_options(tint-remote-compile PRIVATE "-fmodules" "-fcxx-modules")
target_link_options(tint-remote-compile PRIVATE "-framework" "CoreGraphics")
endif()
endif()

View File

@ -0,0 +1,30 @@
// Copyright 2021 The Tint Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#ifndef TOOLS_SRC_CMD_REMOTE_COMPILE_COMPILE_H_
#define TOOLS_SRC_CMD_REMOTE_COMPILE_COMPILE_H_
#include <string>
/// The return structure of a compile function
struct CompileResult {
/// True if shader compiled
bool success = false;
/// Output of the compiler
std::string output;
};
CompileResult CompileMslUsingMetalAPI(const std::string& src);
#endif // TOOLS_SRC_CMD_REMOTE_COMPILE_COMPILE_H_

View File

@ -0,0 +1,440 @@
// Copyright 2021 The Tint Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include <stdio.h>
#include <fstream>
#include <sstream>
#include <string>
#include <type_traits>
#include <vector>
#include "tools/src/cmd/remote-compile/compile.h"
#include "tools/src/cmd/remote-compile/socket.h"
namespace {
#if 0
#define DEBUG(msg, ...) printf(msg "\n", ##__VA_ARGS__)
#else
#define DEBUG(...)
#endif
/// Print the tool usage, and exit with 1.
void ShowUsage() {
const char* name = "tint-remote-compile";
printf(R"(%s is a tool for compiling a shader on a remote machine
usage as server:
%s -s [-p port-number]
usage as client:
%s [-p port-number] [server-address] shader-file-path
[server-address] can be omitted if the TINT_REMOTE_COMPILE_ADDRESS environment
variable is set.
Alternatively, you can pass xcrun arguments so %s can be used as a
drop-in replacement.
)",
name, name, name, name);
exit(1);
}
/// The protocol version code. Bump each time the protocol changes
constexpr uint32_t kProtocolVersion = 1;
/// Supported shader source languages
enum SourceLanguage {
MSL,
};
/// Stream is a serialization wrapper around a socket
struct Stream {
/// The underlying socket
Socket* const socket;
/// Error state
std::string error;
/// Writes a uint32_t to the socket
Stream operator<<(uint32_t v) {
if (error.empty()) {
Write(&v, sizeof(v));
}
return *this;
}
/// Reads a uint32_t from the socket
Stream operator>>(uint32_t& v) {
if (error.empty()) {
Read(&v, sizeof(v));
}
return *this;
}
/// Writes a std::string to the socket
Stream operator<<(const std::string& v) {
if (error.empty()) {
uint32_t count = static_cast<uint32_t>(v.size());
*this << count;
if (count) {
Write(v.data(), count);
}
}
return *this;
}
/// Reads a std::string from the socket
Stream operator>>(std::string& v) {
uint32_t count = 0;
*this >> count;
if (count) {
std::vector<char> buf(count);
if (Read(buf.data(), count)) {
v = std::string(buf.data(), buf.size());
}
} else {
v.clear();
}
return *this;
}
/// Writes an enum value to the socket
template <typename T>
std::enable_if_t<std::is_enum<T>::value, Stream> operator<<(T e) {
return *this << static_cast<uint32_t>(e);
}
/// Reads an enum value from the socket
template <typename T>
std::enable_if_t<std::is_enum<T>::value, Stream> operator>>(T& e) {
uint32_t v;
*this >> v;
e = static_cast<T>(v);
return *this;
}
private:
bool Write(const void* data, size_t size) {
if (error.empty()) {
if (!socket->Write(data, size)) {
error = "Socket::Write() failed";
}
}
return error.empty();
}
bool Read(void* data, size_t size) {
auto buf = reinterpret_cast<uint8_t*>(data);
while (size > 0 && error.empty()) {
if (auto n = socket->Read(buf, size)) {
if (n > size) {
error = "Socket::Read() returned more bytes than requested";
return false;
}
size -= n;
buf += n;
}
}
return error.empty();
}
};
////////////////////////////////////////////////////////////////////////////////
// Messages
////////////////////////////////////////////////////////////////////////////////
/// Base class for all messages
struct Message {
/// The type of the message
enum class Type {
ConnectionRequest,
ConnectionResponse,
CompileRequest,
CompileResponse,
};
explicit Message(Type ty) : type(ty) {}
Type const type;
};
struct ConnectionResponse : Message { // Server -> Client
ConnectionResponse() : Message(Type::ConnectionResponse) {}
template <typename T>
void Serialize(T&& f) {
f(error);
}
std::string error;
};
struct ConnectionRequest : Message { // Client -> Server
using Response = ConnectionResponse;
explicit ConnectionRequest(uint32_t proto_ver = kProtocolVersion)
: Message(Type::ConnectionRequest), protocol_version(proto_ver) {}
template <typename T>
void Serialize(T&& f) {
f(protocol_version);
}
uint32_t protocol_version;
};
struct CompileResponse : Message { // Server -> Client
CompileResponse() : Message(Type::CompileResponse) {}
template <typename T>
void Serialize(T&& f) {
f(error);
}
std::string error;
};
struct CompileRequest : Message { // Client -> Server
using Response = CompileResponse;
CompileRequest() : Message(Type::CompileRequest) {}
CompileRequest(SourceLanguage lang, std::string src)
: Message(Type::CompileRequest), language(lang), source(src) {}
template <typename T>
void Serialize(T&& f) {
f(language);
f(source);
}
SourceLanguage language;
std::string source;
};
/// Writes the message `m` to the stream `s`
template <typename MESSAGE>
std::enable_if_t<std::is_base_of<Message, MESSAGE>::value, Stream>& operator<<(
Stream& s,
const MESSAGE& m) {
s << m.type;
const_cast<MESSAGE&>(m).Serialize([&s](const auto& value) { s << value; });
return s;
}
/// Reads the message `m` from the stream `s`
template <typename MESSAGE>
std::enable_if_t<std::is_base_of<Message, MESSAGE>::value, Stream>& operator>>(
Stream& s,
MESSAGE& m) {
Message::Type ty;
s >> ty;
if (ty == m.type) {
m.Serialize([&s](auto& value) { s >> value; });
} else {
std::stringstream ss;
ss << "expected message type " << static_cast<int>(m.type) << ", got "
<< static_cast<int>(ty);
s.error = ss.str();
}
return s;
}
/// Writes the request message `req` to the stream `s`, then reads and returns
/// the response message from the same stream.
template <typename REQUEST, typename RESPONSE = typename REQUEST::Response>
RESPONSE Send(Stream& s, const REQUEST& req) {
s << req;
if (s.error.empty()) {
RESPONSE resp;
s >> resp;
if (s.error.empty()) {
return resp;
}
}
return {};
}
} // namespace
bool RunServer(std::string port);
bool RunClient(std::string address, std::string port, std::string file);
int main(int argc, char* argv[]) {
bool run_server = false;
std::string port = "19000";
std::vector<std::string> args;
for (int i = 1; i < argc; i++) {
std::string arg = argv[i];
if (arg == "-s" || arg == "--server") {
run_server = true;
continue;
}
if (arg == "-p" || arg == "--port") {
if (i < argc - 1) {
i++;
port = argv[i];
} else {
printf("expected port number");
exit(1);
}
continue;
}
// xcrun flags are ignored so this executable can be used as a replacement
// for xcrun.
if ((arg == "-x" || arg == "-sdk") && (i < argc - 1)) {
i++;
continue;
}
if (arg == "metal") {
for (; i < argc; i++) {
if (std::string(argv[i]) == "-c") {
break;
}
}
continue;
}
args.emplace_back(arg);
}
bool success = false;
if (run_server) {
success = RunServer(port);
} else {
std::string address;
std::string file;
switch (args.size()) {
case 1:
if (auto* addr = getenv("TINT_REMOTE_COMPILE_ADDRESS")) {
address = addr;
}
file = args[0];
break;
case 2:
address = args[0];
file = args[1];
break;
}
if (address.empty() || file.empty()) {
ShowUsage();
}
success = RunClient(address, port, file);
}
if (!success) {
exit(1);
}
return 0;
}
bool RunServer(std::string port) {
auto server_socket = Socket::Listen("", port.c_str());
if (!server_socket) {
printf("Failed to listen on port %s\n", port.c_str());
return false;
}
printf("Listening on port %s...\n", port.c_str());
while (auto conn = server_socket->Accept()) {
DEBUG("Client connected...");
Stream stream{conn.get()};
{
ConnectionRequest req;
stream >> req;
if (!stream.error.empty()) {
printf("%s\n", stream.error.c_str());
continue;
}
ConnectionResponse resp;
if (req.protocol_version != kProtocolVersion) {
DEBUG("Protocol version mismatch");
resp.error = "Protocol version mismatch";
stream << resp;
continue;
}
stream << resp;
}
DEBUG("Connection established");
{
CompileRequest req;
stream >> req;
if (!stream.error.empty()) {
printf("%s\n", stream.error.c_str());
continue;
}
#ifdef TINT_ENABLE_MSL_COMPILATION_USING_METAL_API
if (req.language == SourceLanguage::MSL) {
auto result = CompileMslUsingMetalAPI(req.source);
CompileResponse resp;
if (!result.success) {
resp.error = result.output;
}
stream << resp;
continue;
}
#endif
CompileResponse resp;
resp.error = "server cannot compile this type of shader";
stream << resp;
}
}
return true;
}
bool RunClient(std::string address, std::string port, std::string file) {
// Read the file
std::ifstream input(file, std::ios::binary);
if (!input) {
printf("Couldn't open '%s'\n", file.c_str());
return false;
}
std::string source((std::istreambuf_iterator<char>(input)),
std::istreambuf_iterator<char>());
constexpr const int timeout_ms = 10000;
DEBUG("Connecting to %s:%s...", address.c_str(), port.c_str());
auto conn = Socket::Connect(address.c_str(), port.c_str(), timeout_ms);
if (!conn) {
printf("Connection failed\n");
return false;
}
Stream stream{conn.get()};
DEBUG("Sending connection request...");
auto conn_resp = Send(stream, ConnectionRequest{kProtocolVersion});
if (!stream.error.empty()) {
printf("%s\n", stream.error.c_str());
return false;
}
if (!conn_resp.error.empty()) {
printf("%s\n", conn_resp.error.c_str());
return false;
}
DEBUG("Connection established. Requesting compile...");
auto comp_resp = Send(stream, CompileRequest{SourceLanguage::MSL, source});
if (!stream.error.empty()) {
printf("%s\n", stream.error.c_str());
return false;
}
if (!comp_resp.error.empty()) {
printf("%s\n", comp_resp.error.c_str());
return false;
}
DEBUG("Compilation successful");
return true;
}

View File

@ -0,0 +1,53 @@
// Copyright 2021 The Tint Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#ifdef TINT_ENABLE_MSL_COMPILATION_USING_METAL_API
@import Metal;
// Disable: error: treating #include as an import of module 'std.string'
#pragma clang diagnostic push
#pragma clang diagnostic ignored "-Wauto-import"
#include "compile.h"
#pragma clang diagnostic pop
CompileResult CompileMslUsingMetalAPI(const std::string& src) {
CompileResult result;
result.success = false;
NSError* error = nil;
id<MTLDevice> device = MTLCreateSystemDefaultDevice();
if (!device) {
result.output = "MTLCreateSystemDefaultDevice returned null";
result.success = false;
return result;
}
NSString* source = [NSString stringWithCString:src.c_str()
encoding:NSUTF8StringEncoding];
id<MTLLibrary> library = [device newLibraryWithSource:source
options:nil
error:&error];
if (!library) {
NSString* output = [error localizedDescription];
result.output = [output UTF8String];
result.success = false;
}
return result;
}
#endif

View File

@ -0,0 +1,190 @@
// Copyright 2020 The Tint Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#ifndef TOOLS_SRC_CMD_REMOTE_COMPILE_RWMUTEX_H_
#define TOOLS_SRC_CMD_REMOTE_COMPILE_RWMUTEX_H_
#include <condition_variable> // NOLINT
#include <mutex> // NOLINT
////////////////////////////////////////////////////////////////////////////////
// RWMutex
////////////////////////////////////////////////////////////////////////////////
/// A RWMutex is a reader/writer mutual exclusion lock.
/// The lock can be held by an arbitrary number of readers or a single writer.
/// Also known as a shared mutex.
class RWMutex {
public:
inline RWMutex() = default;
/// lockReader() locks the mutex for reading.
/// Multiple read locks can be held while there are no writer locks.
inline void lockReader();
/// unlockReader() unlocks the mutex for reading.
inline void unlockReader();
/// lockWriter() locks the mutex for writing.
/// If the lock is already locked for reading or writing, lockWriter blocks
/// until the lock is available.
inline void lockWriter();
/// unlockWriter() unlocks the mutex for writing.
inline void unlockWriter();
private:
RWMutex(const RWMutex&) = delete;
RWMutex& operator=(const RWMutex&) = delete;
int readLocks = 0;
int pendingWriteLocks = 0;
std::mutex mutex;
std::condition_variable cv;
};
void RWMutex::lockReader() {
std::unique_lock<std::mutex> lock(mutex);
readLocks++;
}
void RWMutex::unlockReader() {
std::unique_lock<std::mutex> lock(mutex);
readLocks--;
if (readLocks == 0 && pendingWriteLocks > 0) {
cv.notify_one();
}
}
void RWMutex::lockWriter() {
std::unique_lock<std::mutex> lock(mutex);
if (readLocks > 0) {
pendingWriteLocks++;
cv.wait(lock, [&] { return readLocks == 0; });
pendingWriteLocks--;
}
lock.release(); // Keep lock held
}
void RWMutex::unlockWriter() {
if (pendingWriteLocks > 0) {
cv.notify_one();
}
mutex.unlock();
}
////////////////////////////////////////////////////////////////////////////////
// RLock
////////////////////////////////////////////////////////////////////////////////
/// RLock is a RAII read lock helper for a RWMutex.
class RLock {
public:
/// Constructor.
/// Locks `mutex` with a read-lock for the lifetime of the WLock.
/// @param mutex the mutex
explicit inline RLock(RWMutex& mutex);
/// Destructor.
/// Unlocks the RWMutex.
inline ~RLock();
/// Move constructor
/// @param other the other RLock to move into this RLock.
inline RLock(RLock&& other);
/// Move assignment operator
/// @param other the other RLock to move into this RLock.
/// @returns this RLock so calls can be chained
inline RLock& operator=(RLock&& other);
private:
RLock(const RLock&) = delete;
RLock& operator=(const RLock&) = delete;
RWMutex* m;
};
RLock::RLock(RWMutex& mutex) : m(&mutex) {
m->lockReader();
}
RLock::~RLock() {
if (m != nullptr) {
m->unlockReader();
}
}
RLock::RLock(RLock&& other) {
m = other.m;
other.m = nullptr;
}
RLock& RLock::operator=(RLock&& other) {
m = other.m;
other.m = nullptr;
return *this;
}
////////////////////////////////////////////////////////////////////////////////
// WLock
////////////////////////////////////////////////////////////////////////////////
/// WLock is a RAII write lock helper for a RWMutex.
class WLock {
public:
/// Constructor.
/// Locks `mutex` with a write-lock for the lifetime of the WLock.
/// @param mutex the mutex
explicit inline WLock(RWMutex& mutex);
/// Destructor.
/// Unlocks the RWMutex.
inline ~WLock();
/// Move constructor
/// @param other the other WLock to move into this WLock.
inline WLock(WLock&& other);
/// Move assignment operator
/// @param other the other WLock to move into this WLock.
/// @returns this WLock so calls can be chained
inline WLock& operator=(WLock&& other);
private:
WLock(const WLock&) = delete;
WLock& operator=(const WLock&) = delete;
RWMutex* m;
};
WLock::WLock(RWMutex& mutex) : m(&mutex) {
m->lockWriter();
}
WLock::~WLock() {
if (m != nullptr) {
m->unlockWriter();
}
}
WLock::WLock(WLock&& other) {
m = other.m;
other.m = nullptr;
}
WLock& WLock::operator=(WLock&& other) {
m = other.m;
other.m = nullptr;
return *this;
}
#endif // TOOLS_SRC_CMD_REMOTE_COMPILE_RWMUTEX_H_

View File

@ -0,0 +1,310 @@
// Copyright 2021 The Tint Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "tools/src/cmd/remote-compile/socket.h"
#include "tools/src/cmd/remote-compile/rwmutex.h"
#if defined(_WIN32)
#include <winsock2.h>
#include <ws2tcpip.h>
#else
#include <netdb.h>
#include <netinet/in.h>
#include <netinet/tcp.h>
#include <sys/select.h>
#include <sys/socket.h>
#include <sys/time.h>
#include <unistd.h>
#endif
#if defined(_WIN32)
#include <atomic>
namespace {
std::atomic<int> wsaInitCount = {0};
} // anonymous namespace
#else
#include <fcntl.h>
namespace {
using SOCKET = int;
} // anonymous namespace
#endif
namespace {
constexpr SOCKET InvalidSocket = static_cast<SOCKET>(-1);
void init() {
#if defined(_WIN32)
if (wsaInitCount++ == 0) {
WSADATA winsockData;
(void)WSAStartup(MAKEWORD(2, 2), &winsockData);
}
#endif
}
void term() {
#if defined(_WIN32)
if (--wsaInitCount == 0) {
WSACleanup();
}
#endif
}
bool setBlocking(SOCKET s, bool blocking) {
#if defined(_WIN32)
u_long mode = blocking ? 0 : 1;
return ioctlsocket(s, FIONBIO, &mode) == NO_ERROR;
#else
auto arg = fcntl(s, F_GETFL, nullptr);
if (arg < 0) {
return false;
}
arg = blocking ? (arg & ~O_NONBLOCK) : (arg | O_NONBLOCK);
return fcntl(s, F_SETFL, arg) >= 0;
#endif
}
bool errored(SOCKET s) {
if (s == InvalidSocket) {
return true;
}
char error = 0;
socklen_t len = sizeof(error);
getsockopt(s, SOL_SOCKET, SO_ERROR, &error, &len);
return error != 0;
}
class Impl : public Socket {
public:
static std::shared_ptr<Impl> create(const char* address, const char* port) {
init();
addrinfo hints = {};
hints.ai_family = AF_INET;
hints.ai_socktype = SOCK_STREAM;
hints.ai_protocol = IPPROTO_TCP;
hints.ai_flags = AI_PASSIVE;
addrinfo* info = nullptr;
auto err = getaddrinfo(address, port, &hints, &info);
#if !defined(_WIN32)
if (err) {
printf("getaddrinfo(%s, %s) error: %s\n", address, port,
gai_strerror(err));
}
#endif
if (info) {
auto socket =
::socket(info->ai_family, info->ai_socktype, info->ai_protocol);
auto out = std::make_shared<Impl>(info, socket);
out->setOptions();
return out;
}
freeaddrinfo(info);
term();
return nullptr;
}
explicit Impl(SOCKET socket) : info(nullptr), s(socket) {}
Impl(addrinfo* info, SOCKET socket) : info(info), s(socket) {}
~Impl() {
freeaddrinfo(info);
Close();
term();
}
template <typename FUNCTION>
void lock(FUNCTION&& f) {
RLock l(mutex);
f(s, info);
}
void setOptions() {
RLock l(mutex);
if (s == InvalidSocket) {
return;
}
int enable = 1;
#if !defined(_WIN32)
// Prevent sockets lingering after process termination, causing
// reconnection issues on the same port.
setsockopt(s, SOL_SOCKET, SO_REUSEADDR, reinterpret_cast<char*>(&enable),
sizeof(enable));
struct {
int l_onoff; /* linger active */
int l_linger; /* how many seconds to linger for */
} linger = {false, 0};
setsockopt(s, SOL_SOCKET, SO_LINGER, reinterpret_cast<char*>(&linger),
sizeof(linger));
#endif // !defined(_WIN32)
// Enable TCP_NODELAY.
setsockopt(s, IPPROTO_TCP, TCP_NODELAY, reinterpret_cast<char*>(&enable),
sizeof(enable));
}
bool IsOpen() override {
{
RLock l(mutex);
if ((s != InvalidSocket) && !errored(s)) {
return true;
}
}
WLock lock(mutex);
s = InvalidSocket;
return false;
}
void Close() override {
{
RLock l(mutex);
if (s != InvalidSocket) {
#if defined(_WIN32)
closesocket(s);
#else
::shutdown(s, SHUT_RDWR);
#endif
}
}
WLock l(mutex);
if (s != InvalidSocket) {
#if !defined(_WIN32)
::close(s);
#endif
s = InvalidSocket;
}
}
size_t Read(void* buffer, size_t bytes) override {
RLock lock(mutex);
if (s == InvalidSocket) {
return 0;
}
auto len =
recv(s, reinterpret_cast<char*>(buffer), static_cast<int>(bytes), 0);
return (len < 0) ? 0 : len;
}
bool Write(const void* buffer, size_t bytes) override {
RLock lock(mutex);
if (s == InvalidSocket) {
return false;
}
if (bytes == 0) {
return true;
}
return ::send(s, reinterpret_cast<const char*>(buffer),
static_cast<int>(bytes), 0) > 0;
}
std::shared_ptr<Socket> Accept() override {
std::shared_ptr<Impl> out;
lock([&](SOCKET socket, const addrinfo*) {
if (socket != InvalidSocket) {
init();
out = std::make_shared<Impl>(::accept(socket, 0, 0));
out->setOptions();
}
});
return out;
}
private:
addrinfo* const info;
SOCKET s = InvalidSocket;
RWMutex mutex;
};
} // anonymous namespace
std::shared_ptr<Socket> Socket::Listen(const char* address, const char* port) {
auto impl = Impl::create(address, port);
if (!impl) {
return nullptr;
}
impl->lock([&](SOCKET socket, const addrinfo* info) {
if (bind(socket, info->ai_addr, static_cast<int>(info->ai_addrlen)) != 0) {
impl.reset();
return;
}
if (listen(socket, 0) != 0) {
impl.reset();
return;
}
});
return impl;
}
std::shared_ptr<Socket> Socket::Connect(const char* address,
const char* port,
uint32_t timeoutMillis) {
auto impl = Impl::create(address, port);
if (!impl) {
return nullptr;
}
std::shared_ptr<Socket> out;
impl->lock([&](SOCKET socket, const addrinfo* info) {
if (socket == InvalidSocket) {
return;
}
if (timeoutMillis == 0) {
if (::connect(socket, info->ai_addr,
static_cast<int>(info->ai_addrlen)) == 0) {
out = impl;
}
return;
}
if (!setBlocking(socket, false)) {
return;
}
auto res =
::connect(socket, info->ai_addr, static_cast<int>(info->ai_addrlen));
if (res == 0) {
if (setBlocking(socket, true)) {
out = impl;
}
} else {
const auto microseconds = timeoutMillis * 1000;
fd_set fdset;
FD_ZERO(&fdset);
FD_SET(socket, &fdset);
timeval tv;
tv.tv_sec = microseconds / 1000000;
tv.tv_usec = microseconds - static_cast<uint32_t>(tv.tv_sec * 1000000);
res = select(static_cast<int>(socket + 1), nullptr, &fdset, nullptr, &tv);
if (res > 0 && !errored(socket) && setBlocking(socket, true)) {
out = impl;
}
}
});
if (!out) {
return nullptr;
}
return out->IsOpen() ? out : nullptr;
}

View File

@ -0,0 +1,71 @@
// Copyright 2021 The Tint Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#ifndef TOOLS_SRC_CMD_REMOTE_COMPILE_SOCKET_H_
#define TOOLS_SRC_CMD_REMOTE_COMPILE_SOCKET_H_
#include <atomic>
#include <memory>
/// Socket provides an OS abstraction to a TCP socket.
class Socket {
public:
/// Connects to the given TCP address and port.
/// @param address the target socket address
/// @param port the target socket port
/// @param timeoutMillis the timeout for the connection attempt.
/// If timeoutMillis is non-zero and no connection was made before
/// timeoutMillis milliseconds, then nullptr is returned.
/// @returns the connected Socket, or nullptr on failure
static std::shared_ptr<Socket> Connect(const char* address,
const char* port,
uint32_t timeoutMillis);
/// Begins listening for connections on the given TCP address and port.
/// Call Accept() on the returned Socket to block and wait for a connection.
/// @param address the socket address to listen on. Use "localhost" for
/// connections from only this machine, or an empty string to allow
/// connections from any incoming address.
/// @param port the socket port to listen on
/// @returns the Socket that listens for connections
static std::shared_ptr<Socket> Listen(const char* address, const char* port);
/// Attempts to read at most `n` bytes into buffer, returning the actual
/// number of bytes read.
/// read() will block until the socket is closed or at least one byte is read.
/// @param buffer the output buffer. Must be at least `n` bytes in size.
/// @param n the maximum number of bytes to read
/// @return the number of bytes read, or 0 if the socket was closed
virtual size_t Read(void* buffer, size_t n) = 0;
/// Writes `n` bytes from buffer into the socket.
/// @param buffer the source data buffer. Must be at least `n` bytes in size.
/// @param n the number of bytes to read from `buffer`
/// @returns true on success, or false if there was an error or the socket was
/// closed.
virtual bool Write(const void* buffer, size_t n) = 0;
/// @returns true if the socket has not been closed.
virtual bool IsOpen() = 0;
/// Closes the socket.
virtual void Close() = 0;
/// Blocks for a connection to be made to the listening port, or for the
/// Socket to be closed.
/// @returns a pointer to the next established incoming connection
virtual std::shared_ptr<Socket> Accept() = 0;
};
#endif // TOOLS_SRC_CMD_REMOTE_COMPILE_SOCKET_H_