diff --git a/.clang-format b/.clang-format index 06ea346..dd860a0 100644 --- a/.clang-format +++ b/.clang-format @@ -1,4 +1,6 @@ --- -Language: Cpp -BasedOnStyle: Google +Language: Cpp +BasedOnStyle: Google +DerivePointerAlignment: false +PointerAlignment: Left ... diff --git a/.gitmodules b/.gitmodules index 7b743ea..ee4ef6b 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,3 +1,12 @@ [submodule "oss-internship-2020/sapi_lodepng/lodepng"] path = oss-internship-2020/sapi_lodepng/lodepng url = https://github.com/lvandeve/lodepng +[submodule "oss-internship-2020/openjpeg/openjpeg"] + path = oss-internship-2020/openjpeg/openjpeg + url = https://github.com/uclouvain/openjpeg.git +[submodule "oss-internship-2020/pffft/master"] + path = oss-internship-2020/pffft/master + url = https://bitbucket.org/jpommier/pffft/src/master/ +[submodule "oss-internship-2020/curl/curl_wrapper/curl"] + path = oss-internship-2020/curl/curl_wrapper/curl + url = https://github.com/curl/curl diff --git a/CMakeLists.txt b/CMakeLists.txt index 7c993f4..20b732e 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -14,6 +14,16 @@ cmake_minimum_required(VERSION 3.12) +# Fix Ninja generator output to not rebuild entire sub-trees needlessly. +if(CMAKE_GENERATOR MATCHES "Ninja") + file(WRITE "${CMAKE_BINARY_DIR}/UserMakeRulesOverride.cmake" + "string(REPLACE \"-MD\" \"-MMD\" CMAKE_DEPFILE_FLAGS_C \"\${CMAKE_DEPFILE_FLAGS_C}\")\n" + "string(REPLACE \"-MD\" \"-MMD\" CMAKE_DEPFILE_FLAGS_CXX \"\${CMAKE_DEPFILE_FLAGS_CXX}\")\n" + ) + set(CMAKE_USER_MAKE_RULES_OVERRIDE + "${CMAKE_BINARY_DIR}/UserMakeRulesOverride.cmake" CACHE INTERNAL "") +endif() + project(SandboxedAPI C CXX ASM) # SAPI-wide setting for the language level @@ -31,15 +41,8 @@ include(SapiDeps) include(SapiUtil) include(SapiBuildDefs) -# Fix Ninja generator output to not rebuild entire sub-trees needlessly. -if(CMAKE_GENERATOR MATCHES "Ninja") - file(WRITE "${SAPI_BINARY_DIR}/UserMakeRulesOverride.cmake" - "STRING(REPLACE \"-MD\" \"-MMD\" CMAKE_DEPFILE_FLAGS_C \"\${CMAKE_DEPFILE_FLAGS_C}\")\n" - "STRING(REPLACE \"-MD\" \"-MMD\" CMAKE_DEPFILE_FLAGS_CXX \"\${CMAKE_DEPFILE_FLAGS_CXX}\")\n" - ) - set(CMAKE_USER_MAKE_RULES_OVERRIDE - "${SAPI_BINARY_DIR}/UserMakeRulesOverride.cmake" CACHE INTERNAL "") -endif() +# Allow the header generator to auto-configure include paths +set(CMAKE_EXPORT_COMPILE_COMMANDS ON) if (SAPI_FORCE_COLOR_OUTPUT) if(CMAKE_CXX_COMPILER_ID STREQUAL "GNU") # GCC diff --git a/cmake/SapiBuildDefs.cmake b/cmake/SapiBuildDefs.cmake index 5e75e59..5b9ce28 100644 --- a/cmake/SapiBuildDefs.cmake +++ b/cmake/SapiBuildDefs.cmake @@ -123,7 +123,7 @@ function(add_sapi_library) endif() # Interface - list_join(_sapi_FUNCTIONS "," _sapi_funcs) + list(JOIN _sapi_FUNCTIONS "," _sapi_funcs) foreach(src IN LISTS _sapi_INPUTS) get_filename_component(src "${src}" ABSOLUTE) list(APPEND _sapi_full_inputs "${src}") @@ -132,7 +132,6 @@ function(add_sapi_library) set(_sapi_embed_dir "${CMAKE_CURRENT_BINARY_DIR}") set(_sapi_embed_name "${_sapi_NAME}") endif() - # TODO(cblichmann): Implement sapi_isystem if(SAPI_ENABLE_GENERATOR) add_custom_command( OUTPUT "${_sapi_gen_header}" @@ -149,9 +148,14 @@ function(add_sapi_library) VERBATIM ) else() - list_join(_sapi_full_inputs "," _sapi_full_inputs) + set(_sapi_isystem "${_sapi_NAME}.isystem") + list(JOIN _sapi_full_inputs "," _sapi_full_inputs) add_custom_command( - OUTPUT "${_sapi_gen_header}" + OUTPUT "${_sapi_gen_header}" "${_sapi_isystem}" + COMMAND sh -c + "${CMAKE_CXX_COMPILER} -E -x c++ -v /dev/null 2>&1 | \ + awk '/> search starts here:/{f=1;next}/^End of search/{f=0}f{print $1}' \ + > \"${_sapi_isystem}\"" COMMAND "${SAPI_PYTHON3_EXECUTABLE}" -B "${SAPI_SOURCE_DIR}/sandboxed_api/tools/generator2/sapi_generator.py" "--sapi_name=${_sapi_LIBRARY_NAME}" @@ -160,8 +164,10 @@ function(add_sapi_library) "--sapi_embed_name=${_sapi_embed_name}" "--sapi_functions=${_sapi_funcs}" "--sapi_ns=${_sapi_NAMESPACE}" + "--sapi_isystem=${_sapi_isystem}" "--sapi_in=${_sapi_full_inputs}" COMMENT "Generating interface" + VERBATIM ) endif() diff --git a/cmake/SapiUtil.cmake b/cmake/SapiUtil.cmake index 8773f59..f4e891f 100644 --- a/cmake/SapiUtil.cmake +++ b/cmake/SapiUtil.cmake @@ -35,16 +35,6 @@ function(create_directory_symlink SOURCE DESTINATION) endif() endfunction() -# Implements list(JOIN ...) for CMake < 3.12. -function(list_join LIST SEP OUTPUT) - foreach(item IN LISTS ${LIST}) - set(_concat "${_concat}${SEP}${item}") - endforeach() - string(LENGTH "${SEP}" _len) - string(SUBSTRING "${_concat}" ${_len} -1 _concat) - set(${OUTPUT} "${_concat}" PARENT_SCOPE) -endfunction() - # Helper function that behaves just like Protobuf's protobuf_generate_cpp(), # except that it strips import paths. This is necessary, because CMake's # protobuf rules don't work well with imports across different directories. diff --git a/cmake/abseil/CMakeLists.txt.in b/cmake/abseil/CMakeLists.txt.in index 2484f6e..01aacba 100644 --- a/cmake/abseil/CMakeLists.txt.in +++ b/cmake/abseil/CMakeLists.txt.in @@ -18,7 +18,7 @@ project(absl-download NONE) include(ExternalProject) ExternalProject_Add(absl GIT_REPOSITORY https://github.com/abseil/abseil-cpp - GIT_TAG 6e18c7115df9b7ca0987cc346b1b1d4b3cc829b3 # 2020-04-28 + GIT_TAG 0e9921b75a0fdd639a504ec8443fc1fe801becd7 # 2020-09-02 SOURCE_DIR "${CMAKE_BINARY_DIR}/absl-src" BINARY_DIR "${CMAKE_BINARY_DIR}/absl-build" CONFIGURE_COMMAND "" diff --git a/oss-internship-2020/curl/.gitignore b/oss-internship-2020/curl/.gitignore new file mode 100644 index 0000000..567609b --- /dev/null +++ b/oss-internship-2020/curl/.gitignore @@ -0,0 +1 @@ +build/ diff --git a/oss-internship-2020/curl/CMakeLists.txt b/oss-internship-2020/curl/CMakeLists.txt new file mode 100644 index 0000000..8d2be38 --- /dev/null +++ b/oss-internship-2020/curl/CMakeLists.txt @@ -0,0 +1,144 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +cmake_minimum_required(VERSION 3.16) + +project(libcurl_sandbox) + +set(CMAKE_CXX_STANDARD 17) +set(CMAKE_CXX_STANDARD_REQUIRED True) + +option(CURL_SAPI_ENABLE_EXAMPLES "" ON) +option(CURL_SAPI_ENABLE_TESTS "" ON) + +# Add callbacks used by examples and tests +if (CURL_SAPI_ENABLE_EXAMPLES OR CURL_SAPI_ENABLE_TESTS) + list(APPEND CURL_SAPI_CALLBACKS + "${CMAKE_CURRENT_SOURCE_DIR}/callbacks/callbacks.h" + "${CMAKE_CURRENT_SOURCE_DIR}/callbacks/callbacks.cc" + ) +endif() + +# Add folder containing the non-sandboxed custom curl library +add_subdirectory(curl_wrapper) + +# Setup Sandboxed API +set(SAPI_ROOT "" CACHE PATH "Path to the Sandboxed API source tree") +set(SAPI_ENABLE_EXAMPLES ${CURL_SAPI_ENABLE_EXAMPLES} CACHE BOOL "" FORCE) +set(SAPI_ENABLE_TESTS ${CURL_SAPI_ENABLE_TESTS} CACHE BOOL "" FORCE) +add_subdirectory( + "${SAPI_ROOT}" + "${CMAKE_BINARY_DIR}/sandboxed-api-build" + EXCLUDE_FROM_ALL +) + +# Generate SAPI header +add_sapi_library(curl_sapi + + # List of all the methods in https://curl.haxx.se/libcurl/c/allfuncs.html + # Some are added or modified because the original ones are not supported + # by Sandboxed API (details can be found in curl_wrapper.h) + FUNCTIONS curl_easy_cleanup + curl_easy_duphandle + curl_easy_escape + curl_easy_getinfo + curl_easy_getinfo_ptr + curl_easy_init + curl_easy_pause + curl_easy_perform + curl_easy_recv + curl_easy_reset + curl_easy_send + curl_easy_setopt + curl_easy_setopt_ptr + curl_easy_setopt_long + curl_easy_setopt_curl_off_t + curl_easy_strerror + curl_easy_unescape + curl_easy_upkeep + curl_free + curl_getdate_sapi + curl_global_cleanup + curl_global_init + curl_global_init_mem + curl_global_sslset + curl_mime_addpart + curl_mime_data + curl_mime_data_cb + curl_mime_encoder + curl_mime_filedata + curl_mime_filename + curl_mime_free + curl_mime_headers + curl_mime_init + curl_mime_name + curl_mime_subparts + curl_mime_type + curl_multi_add_handle + curl_multi_assign + curl_multi_cleanup + curl_multi_fdset_sapi + curl_multi_info_read + curl_multi_init + curl_multi_perform + curl_multi_remove_handle + curl_multi_setopt + curl_multi_setopt_ptr + curl_multi_setopt_long + curl_multi_setopt_curl_off_t + curl_multi_socket_action + curl_multi_strerror + curl_multi_timeout + curl_multi_poll_sapi + curl_multi_wait_sapi + curl_multi_wakeup + curl_share_init + curl_share_setopt + curl_share_setopt_ptr + curl_share_setopt_long + curl_share_strerror + curl_slist_append + curl_slist_free_all + curl_url + curl_url_cleanup + curl_url_dup + curl_url_get + curl_url_set + curl_version + curl_version_info + + INPUTS curl_wrapper/curl/include/curl/curl.h + curl_wrapper/curl_wrapper.h + + LIBRARY curl_wrapper_and_callbacks + + LIBRARY_NAME Curl + + NAMESPACE "" +) + +# Include generated SAPI header +target_include_directories(curl_sapi INTERFACE + "${PROJECT_BINARY_DIR}" +) + +# Add examples +if (CURL_SAPI_ENABLE_EXAMPLES) + add_subdirectory(examples) +endif() + +# Add tests +if (CURL_SAPI_ENABLE_TESTS) + add_subdirectory(tests) +endif() diff --git a/oss-internship-2020/curl/README.md b/oss-internship-2020/curl/README.md new file mode 100644 index 0000000..3000838 --- /dev/null +++ b/oss-internship-2020/curl/README.md @@ -0,0 +1,112 @@ +# LibCurl Sandbox + +This library is a sandboxed version of curl's C API, +[libcurl](https://curl.haxx.se/libcurl/c/), implemented using Sandboxed API. + +## Setup + +The repository can be cloned using: `git clone --recursive [URL to this repo]` +The `--recursive` flag ensures that submodules are also cloned. + +Alternatively, if the repository has already been cloned but the submodules have +not, these can be cloned using: `git submodule update --init --recursive` + +The full list of Sandboxed API dependencies can be found on +[Sandboxed API Getting Started page](https://developers.google.com/sandboxed-api/docs/getting-started). + +The following commands, used from the current `curl/` directory, build the +library: `mkdir -p build cd build cmake .. -G Ninja -D SAPI_ROOT=[path to +sandboxed-api] cmake --build .` + +## Implementation details + +All of libcurl's methods are supported by the library. However, a few of these +have different signatures defined in the sandboxed header `custom_curl.h`, which +wraps and extends libcurl. + +This is necessary because Sandboxed API sandboxes most of libcurl correctly, but +encounters some issues when sandboxing a few methods. The simplest solution is +wrapping these methods into wrapper methods that accomplish the same tasks but +can also be sandboxed. + +The next sections describe the issues encountered and contain some information +on the signatures of the wrapper methods solving these issues. + +#### Variadic methods + +Variadic methods are currently not supported by Sandboxed API. To solve this, +these methods are defined with an additional explicit parameter in +`custom_curl.h`. + +The methods are: - `curl_easy_setopt`. Use `curl_easy_setopt_ptr`, +`curl_easy_setopt_long` or `curl_easy_setopt_curl_off_t` instead. - +`curl_easy_getinfo`. Use `curl_easy_getinfo_ptr` instead. - `curl_multi_setopt`. +Use `curl_multi_setopt_ptr`, `curl_multi_setopt_long` or +`curl_multi_setopt_curl_off_t` instead. - `curl_share_setopt`. Use +`curl_share_setopt_ptr` or `curl_share_setopt_long` instead + +#### Methods with incomplete array arguments + +Incomplete array arguments are currently not supported by Sandboxed API. To +solve this, methods taking an incomplete array argument have a wrapper in +`custom_curl.h`, and take a pointer as the argument. + +The methods are: - `curl_multi_poll`. Use `curl_multi_poll_sapi` instead. - +`curl_multi_wait`. Use `curl_multi_wait_sapi` instead. + +#### Methods with conflicts on the generated header + +Some methods create conflicts on the generated header because of redefined +`#define` directives from files included by the header. To solve this, the +conflicting types and methods are redefined in `custom_curl.h`. + +The types are: - `time_t`. Use `time_t_sapi` instead. - `fd_set`. Use +`fd_set_sapi` instead. + +The methods are: - `curl_getdate`. Use `curl_getdate_sapi` instead. - +`curl_multi_fdset`. Use `curl_multi_fdset_sapi` instead. + +#### Function pointers + +The functions whose pointers will be passed to the library's methods +(*callbacks*) can't be implemented in the files making use of the library, but +must be in other files. These files must be compiled together with the library, +and this is done by adding their absolute path to the cmake variable +`CURL_SAPI_CALLBACKS`. + +The pointers can then be obtained using an `RPCChannel` object, as shown in +`example2.cc`. + +## Examples + +The `examples` directory contains the sandboxed versions of example source codes +taken from [this page](https://curl.haxx.se/libcurl/c/example.html) on curl's +website. More information about each example can be found in the examples' +[README](examples/README.md). + +To build these examples when building the library, the cmake variable +`CURL_SAPI_ENABLE_EXAMPLES` must be set to `ON`. This enables Sandboxed API +examples as well. + +## Policy + +The `sandbox.h` file contains a policy allowing all is necessary for libcurl to +perform simple requests. It is used by all the examples, except by example3. +This example needs some additional policies and files in its namespace (since it +uses HTTPS), and the file `example3.cc` shows how to easily extend an existing +policy. + +## Testing + +The `tests` folder contains some test cases created using Google Test. The class +`CurlTestUtils` is used to facilitate some tasks that all test cases need, +including the setup of a mock local server on which test requests are performed. + +To build these tests when building the library, the cmake variable +`CURL_SAPI_ENABLE_TESTS` must be set to `ON`. This enables Sandboxed API tests +as well. + +## Callbacks + +The `callbacks.h` and `callbacks.cc` files implement all the callbacks used by +examples and tests. diff --git a/oss-internship-2020/curl/callbacks/callbacks.cc b/oss-internship-2020/curl/callbacks/callbacks.cc new file mode 100644 index 0000000..015e66c --- /dev/null +++ b/oss-internship-2020/curl/callbacks/callbacks.cc @@ -0,0 +1,37 @@ +// Copyright 2020 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "callbacks.h" // NOLINT(build/include) + +#include +#include + +#include "sandboxed_api/vars.h" + +size_t WriteToMemory(char* contents, size_t size, size_t num_bytes, + void* userp) { + size_t real_size = size * num_bytes; + auto* mem = static_cast(userp); + + char* ptr = static_cast(realloc(mem->data, mem->size + real_size + 1)); + if (ptr == nullptr) return 0; + + mem->data = ptr; + auto data = static_cast(mem->data); + memcpy(&(data[mem->size]), contents, real_size); + mem->size += real_size; + data[mem->size] = 0; + + return real_size; +} diff --git a/oss-internship-2020/curl/callbacks/callbacks.h b/oss-internship-2020/curl/callbacks/callbacks.h new file mode 100644 index 0000000..306e2eb --- /dev/null +++ b/oss-internship-2020/curl/callbacks/callbacks.h @@ -0,0 +1,24 @@ +// Copyright 2020 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef TESTS_CALLBACKS_H +#define TESTS_CALLBACKS_H + +#include + +// Append contents to the string stored by userp, which is a sapi::LenValStruct* +extern "C" size_t WriteToMemory(char* contents, size_t size, size_t num_bytes, + void* userp); + +#endif // TESTS_CALLBACKS_H diff --git a/oss-internship-2020/curl/curl_wrapper/CMakeLists.txt b/oss-internship-2020/curl/curl_wrapper/CMakeLists.txt new file mode 100644 index 0000000..78d2526 --- /dev/null +++ b/oss-internship-2020/curl/curl_wrapper/CMakeLists.txt @@ -0,0 +1,37 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Wrapper library including curl, wrappers for some methods and callbacks +# The CURL_SAPI_CALLBACKS variable should contain the absolute paths of +# all the files implementing the callbacks +add_library(curl_wrapper_and_callbacks OBJECT + curl_wrapper.h + curl_wrapper.cc + "${CURL_SAPI_CALLBACKS}" +) +set_target_properties(curl_wrapper_and_callbacks + PROPERTIES LINKER_LANGUAGE C +) + +# Flags needed to build curl statically +set(CURL_HIDDEN_SYMBOLS OFF) +set(BUILD_SHARED_LIBS OFF) + +# Link the wrapper to the original curl library (testing is disabled in curl) +set(BUILD_TESTING OFF) +add_subdirectory(curl) +target_link_libraries(curl_wrapper_and_callbacks + CURL::libcurl + sapi::sapi +) diff --git a/oss-internship-2020/curl/curl_wrapper/curl_wrapper.cc b/oss-internship-2020/curl/curl_wrapper/curl_wrapper.cc new file mode 100644 index 0000000..6105fbe --- /dev/null +++ b/oss-internship-2020/curl/curl_wrapper/curl_wrapper.cc @@ -0,0 +1,86 @@ +// Copyright 2020 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "curl_wrapper.h" // NOLINT(build/include) + +CURLcode curl_easy_setopt_ptr(CURL* handle, CURLoption option, + void* parameter) { + return curl_easy_setopt(handle, option, parameter); +} + +CURLcode curl_easy_setopt_long(CURL* handle, CURLoption option, + long parameter) { + return curl_easy_setopt(handle, option, parameter); +} + +CURLcode curl_easy_setopt_curl_off_t(CURL* handle, CURLoption option, + curl_off_t parameter) { + return curl_easy_setopt(handle, option, parameter); +} + +CURLcode curl_easy_getinfo_ptr(CURL* handle, CURLINFO option, void* parameter) { + return curl_easy_getinfo(handle, option, parameter); +} + +time_t_sapi curl_getdate_sapi(char* datestring, time_t_sapi* now) { + return curl_getdate(datestring, now); +} + +CURLMcode curl_multi_fdset_sapi(CURLM* multi_handle, fd_set_sapi* read_fd_set, + fd_set_sapi* write_fd_set, + fd_set_sapi* exc_fd_set, int* max_fd) { + return curl_multi_fdset(multi_handle, read_fd_set, write_fd_set, exc_fd_set, + max_fd); +} + +CURLMcode curl_multi_setopt_ptr(CURLM* handle, CURLMoption option, + void* parameter) { + return curl_multi_setopt(handle, option, parameter); +} + +CURLMcode curl_multi_setopt_long(CURLM* handle, CURLMoption option, + long parameter) { + return curl_multi_setopt(handle, option, parameter); +} + +CURLMcode curl_multi_setopt_curl_off_t(CURLM* handle, CURLMoption option, + curl_off_t parameter) { + return curl_multi_setopt(handle, option, parameter); +} + +CURLMcode curl_multi_poll_sapi(CURLM* multi_handle, + struct curl_waitfd* extra_fds, + unsigned int extra_nfds, int timeout_ms, + int* numfds) { + return curl_multi_poll(multi_handle, extra_fds, extra_nfds, timeout_ms, + numfds); +} + +CURLMcode curl_multi_wait_sapi(CURLM* multi_handle, + struct curl_waitfd* extra_fds, + unsigned int extra_nfds, int timeout_ms, + int* numfds) { + return curl_multi_wait(multi_handle, extra_fds, extra_nfds, timeout_ms, + numfds); +} + +CURLSHcode curl_share_setopt_ptr(CURLSH* handle, CURLSHoption option, + void* parameter) { + return curl_share_setopt(handle, option, parameter); +} + +CURLSHcode curl_share_setopt_long(CURLSH* handle, CURLSHoption option, + long parameter) { + return curl_share_setopt(handle, option, parameter); +} diff --git a/oss-internship-2020/curl/curl_wrapper/curl_wrapper.h b/oss-internship-2020/curl/curl_wrapper/curl_wrapper.h new file mode 100644 index 0000000..4ef612b --- /dev/null +++ b/oss-internship-2020/curl/curl_wrapper/curl_wrapper.h @@ -0,0 +1,82 @@ +// Copyright 2020 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Wrapper for curl library + +#ifndef CURL_WRAPPER_H +#define CURL_WRAPPER_H + +#include + +extern "C" { + +// The wrapper method is needed to make the variadic argument explicit +CURLcode curl_easy_setopt_ptr(CURL* handle, CURLoption option, void* parameter); + +// The wrapper method is needed to make the variadic argument explicit +CURLcode curl_easy_setopt_long(CURL* handle, CURLoption option, long parameter); + +// The wrapper method is needed to make the variadic argument explicit +CURLcode curl_easy_setopt_curl_off_t(CURL* handle, CURLoption option, + curl_off_t parameter); + +// The wrapper method is needed to make the variadic argument explicit +CURLcode curl_easy_getinfo_ptr(CURL* handle, CURLINFO option, void* parameter); + +// The typedef and wrapper method are needed because the original method has +// some conflicts in curl_sapi.sapi.h +typedef time_t time_t_sapi; +time_t_sapi curl_getdate_sapi(char* datestring, time_t_sapi* now); + +// The typedef and wrapper method are needed because the original method has +// some conflicts in curl_sapi.sapi.h +typedef fd_set fd_set_sapi; +CURLMcode curl_multi_fdset_sapi(CURLM* multi_handle, fd_set_sapi* read_fd_set, + fd_set_sapi* write_fd_set, + fd_set_sapi* exc_fd_set, int* max_fd); + +// The wrapper method is needed to make the variadic argument explicit +CURLMcode curl_multi_setopt_ptr(CURLM* handle, CURLMoption option, + void* parameter); + +// The wrapper method is needed to make the variadic argument explicit +CURLMcode curl_multi_setopt_long(CURLM* handle, CURLMoption option, + long parameter); + +// The wrapper method is needed to make the variadic argument explicit +CURLMcode curl_multi_setopt_curl_off_t(CURLM* handle, CURLMoption option, + curl_off_t parameter); + +// The wrapper method is needed because incomplete array type is not supported +CURLMcode curl_multi_poll_sapi(CURLM* multi_handle, + struct curl_waitfd* extra_fds, + unsigned int extra_nfds, int timeout_ms, + int* numfds); + +// The wrapper method is needed because incomplete array type is not supported +CURLMcode curl_multi_wait_sapi(CURLM* multi_handle, + struct curl_waitfd* extra_fds, + unsigned int extra_nfds, int timeout_ms, + int* numfds); + +// The wrapper method is needed to make the variadic argument explicit +CURLSHcode curl_share_setopt_ptr(CURLSH* handle, CURLSHoption option, + void* parameter); + +// The wrapper method is needed to make the variadic argument explicit +CURLSHcode curl_share_setopt_long(CURLSH* handle, CURLSHoption option, + long parameter); +} + +#endif // CURL_WRAPPER_H diff --git a/oss-internship-2020/curl/examples/CMakeLists.txt b/oss-internship-2020/curl/examples/CMakeLists.txt new file mode 100644 index 0000000..9193109 --- /dev/null +++ b/oss-internship-2020/curl/examples/CMakeLists.txt @@ -0,0 +1,76 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# All the examples are sandboxed versions of curl's examples +# (https://curl.haxx.se/libcurl/c/example.html) + +# Example 1: simple.c +add_executable(example1 + example1.cc + ../sandbox.h +) +target_link_libraries(example1 PRIVATE + curl_sapi + sapi::sapi +) + +# Example 2: getinmemory.c +add_executable(example2 + example2.cc + ../sandbox.h +) +target_link_libraries(example2 PRIVATE + curl_sapi + sapi::sapi +) + +# Example 3: simplessl.c +add_executable(example3 + example3.cc + ../sandbox.h +) +target_link_libraries(example3 PRIVATE + curl_sapi + sapi::sapi +) + +# Example 4: multi-poll.c +add_executable(example4 + example4.cc + ../sandbox.h +) +target_link_libraries(example4 PRIVATE + curl_sapi + sapi::sapi +) + +# Example 5: multithread.c +add_executable(example5 + example5.cc + ../sandbox.h +) +target_link_libraries(example5 PRIVATE + curl_sapi + sapi::sapi +) + +# Example 6: simple.c (using transactions) +add_executable(example6 + example6.cc + ../sandbox.h +) +target_link_libraries(example6 PRIVATE + curl_sapi + sapi::sapi +) diff --git a/oss-internship-2020/curl/examples/README.md b/oss-internship-2020/curl/examples/README.md new file mode 100644 index 0000000..d653a47 --- /dev/null +++ b/oss-internship-2020/curl/examples/README.md @@ -0,0 +1,38 @@ +# LibCurl Sandbox Examples + +Each example in this folder is the sandboxed version of a code snippet from +[this page](https://curl.haxx.se/libcurl/c/example.html) on curl's website. +These examples perform some basic tasks using libcurl, and can be useful both to +understand how to use LibCurl Sandbox, but also to get an idea of how regular +and sandboxed code compare to each other. + +This is the list of the examples: + +- **example1**: sandboxed version of + [simple.c](https://curl.haxx.se/libcurl/c/simple.html). Really simple HTTP + request, downloads and prints out the page at + [example.com](http://example.com). +- **example2**: sandboxed version of + [getinmemory.c](https://curl.haxx.se/libcurl/c/getinmemory.html). Same HTTP + request as example1. The difference is that this example uses a callback to + save the page directly in memory. Only the page size is printed out. +- **example3**: sandboxed version of + [simplessl.c](https://curl.haxx.se/libcurl/c/simplessl.html). HTTPS request + of the [example.com](https://example.com) page, using SSL authentication. + This script takes 4 arguments (SSL certificates file, SSL keys file, SSL + keys password and CA certificates files), and prints out the page. +- **example4**: sandboxed version of + [multi-poll.c](https://curl.haxx.se/libcurl/c/multi-poll.html). Same HTTP + request as example1, with the addition of a polling method that can be used + to track the status of the request. The page is printed out after it is + downloaded. +- **example5**: sandboxed version of + [multithread.c](https://curl.haxx.se/libcurl/c/multithread.html). Four HTTP + request of the pages [example.com](http://example.com), + [example.edu](http://example.edu), [example.net](http://example.net) and + [example.org](http://example.org), performed at the same time using + libcurl's multithreading methods. The pages are printed out. +- **example6**: sandboxed version of + [simple.c](https://curl.haxx.se/libcurl/c/simple.html). Performs the same + tasks as example1, but Sandbox API Transactions are used to show how they + can be used to perform a simple request. diff --git a/oss-internship-2020/curl/examples/example1.cc b/oss-internship-2020/curl/examples/example1.cc new file mode 100644 index 0000000..e5ee5fe --- /dev/null +++ b/oss-internship-2020/curl/examples/example1.cc @@ -0,0 +1,78 @@ +// Copyright 2020 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Sandboxed version of simple.c +// Simple HTTP GET request + +#include + +#include "../sandbox.h" // NOLINT(build/include) + +int main(int argc, char* argv[]) { + gflags::ParseCommandLineFlags(&argc, &argv, true); + google::InitGoogleLogging(argv[0]); + + absl::Status status; + + // Initialize sandbox2 and sapi + CurlSapiSandbox sandbox; + status = sandbox.Init(); + if (!status.ok()) { + LOG(FATAL) << "Couldn't initialize Sandboxed API: " << status; + } + CurlApi api(&sandbox); + + // Initialize the curl session + absl::StatusOr curl_handle = api.curl_easy_init(); + if (!curl_handle.ok()) { + LOG(FATAL) << "curl_easy_init failed: " << curl_handle.status(); + } + sapi::v::RemotePtr curl(curl_handle.value()); + if (!curl.GetValue()) LOG(FATAL) << "curl_easy_init failed: curl is NULL"; + + absl::StatusOr curl_code; + + // Specify URL to get + sapi::v::ConstCStr url("http://example.com"); + curl_code = api.curl_easy_setopt_ptr(&curl, CURLOPT_URL, url.PtrBefore()); + if (!curl_code.ok() || curl_code.value() != CURLE_OK) { + LOG(FATAL) << "curl_easy_setopt_ptr failed: " << curl_code.status(); + } + + // Set the library to follow a redirection + curl_code = api.curl_easy_setopt_long(&curl, CURLOPT_FOLLOWLOCATION, 1l); + if (!curl_code.ok() || curl_code.value() != CURLE_OK) { + LOG(FATAL) << "curl_easy_setopt_long failed: " << curl_code.status(); + } + + // Disable authentication of peer certificate + curl_code = api.curl_easy_setopt_long(&curl, CURLOPT_SSL_VERIFYPEER, 0l); + if (!curl_code.ok() || curl_code.value() != CURLE_OK) { + LOG(FATAL) << "curl_easy_setopt_long failed: " << curl_code.status(); + } + + // Perform the request + curl_code = api.curl_easy_perform(&curl); + if (!curl_code.ok() || curl_code.value() != CURLE_OK) { + LOG(FATAL) << "curl_easy_perform failed: " << curl_code.status(); + } + + // Cleanup curl + status = api.curl_easy_cleanup(&curl); + if (!status.ok()) { + LOG(FATAL) << "curl_easy_cleanup failed: " << status; + } + + return EXIT_SUCCESS; +} diff --git a/oss-internship-2020/curl/examples/example2.cc b/oss-internship-2020/curl/examples/example2.cc new file mode 100644 index 0000000..c7abe0c --- /dev/null +++ b/oss-internship-2020/curl/examples/example2.cc @@ -0,0 +1,107 @@ +// Copyright 2020 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Sandboxed version of getinmemory.c +// HTTP GET request using callbacks + +#include +#include + +#include "../sandbox.h" // NOLINT(build/include) + +int main(int argc, char* argv[]) { + gflags::ParseCommandLineFlags(&argc, &argv, true); + google::InitGoogleLogging(argv[0]); + + absl::Status status; + + // Initialize sandbox2 and sapi + CurlSapiSandbox sandbox; + status = sandbox.Init(); + if (!status.ok()) { + LOG(FATAL) << "Couldn't initialize Sandboxed API: " << status; + } + CurlApi api(&sandbox); + + // Generate pointer to WriteMemoryCallback function + void* function_ptr; + status = sandbox.rpc_channel()->Symbol("WriteToMemory", &function_ptr); + if (!status.ok()) { + LOG(FATAL) << "sapi::Sandbox::rpc_channel().Symbol failed: " << status; + } + sapi::v::RemotePtr remote_function_ptr(function_ptr); + + // Initialize the curl session + absl::StatusOr curl_handle = api.curl_easy_init(); + if (!curl_handle.ok()) { + LOG(FATAL) << "curl_easy_init failed: " << curl_handle.status(); + } + sapi::v::RemotePtr curl(curl_handle.value()); + if (!curl.GetValue()) { + LOG(FATAL) << "curl_easy_init failed: curl is NULL"; + } + + absl::StatusOr curl_code; + + // Specify URL to get + sapi::v::ConstCStr url("http://example.com"); + curl_code = api.curl_easy_setopt_ptr(&curl, CURLOPT_URL, url.PtrBefore()); + if (!curl_code.ok() || curl_code.value() != CURLE_OK) { + LOG(FATAL) << "curl_easy_setopt_ptr failed: " << curl_code.status(); + } + + // Set WriteMemoryCallback as the write function + curl_code = api.curl_easy_setopt_ptr(&curl, CURLOPT_WRITEFUNCTION, + &remote_function_ptr); + if (!curl_code.ok() || curl_code.value() != CURLE_OK) { + LOG(FATAL) << "curl_easy_setopt_ptr failed: " << curl_code.status(); + } + + // Pass 'chunk' struct to the callback function + sapi::v::LenVal chunk(0); + curl_code = + api.curl_easy_setopt_ptr(&curl, CURLOPT_WRITEDATA, chunk.PtrBoth()); + if (!curl_code.ok() || curl_code.value() != CURLE_OK) { + LOG(FATAL) << "curl_easy_setopt_ptr failed: " << curl_code.status(); + } + + // Set a user agent + sapi::v::ConstCStr user_agent("libcurl-agent/1.0"); + curl_code = api.curl_easy_setopt_ptr(&curl, CURLOPT_USERAGENT, + user_agent.PtrBefore()); + if (!curl_code.ok() || curl_code.value() != CURLE_OK) { + LOG(FATAL) << "curl_easy_setopt_ptr failed: " << curl_code.status(); + } + + // Perform the request + curl_code = api.curl_easy_perform(&curl); + if (!curl_code.ok() || curl_code.value() != CURLE_OK) { + LOG(FATAL) << "curl_easy_perform failed: " << curl_code.status(); + } + + // Retrieve memory size + status = sandbox.TransferFromSandboxee(&chunk); + if (!status.ok()) { + LOG(FATAL) << "sandbox.TransferFromSandboxee failed: " << status; + } + std::cout << "memory size: " << chunk.GetDataSize() << " bytes" << std::endl; + + // Cleanup curl + status = api.curl_easy_cleanup(&curl); + if (!status.ok()) { + LOG(FATAL) << "curl_easy_cleanup failed: " << status; + } + + return EXIT_SUCCESS; +} diff --git a/oss-internship-2020/curl/examples/example3.cc b/oss-internship-2020/curl/examples/example3.cc new file mode 100644 index 0000000..1a73c93 --- /dev/null +++ b/oss-internship-2020/curl/examples/example3.cc @@ -0,0 +1,166 @@ +// Copyright 2020 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Sandboxed version of simplessl.c +// HTTPS GET request + +#include + +#include "../sandbox.h" // NOLINT(build/include) + +class CurlSapiSandboxEx3 : public CurlSapiSandbox { + public: + CurlSapiSandboxEx3(std::string ssl_certificate, std::string ssl_key, + std::string ca_certificates) + : ssl_certificate(ssl_certificate), + ssl_key(ssl_key), + ca_certificates(ca_certificates) {} + + private: + std::unique_ptr ModifyPolicy( + sandbox2::PolicyBuilder*) override { + // Add the syscalls and files missing in CurlSandbox to a new PolicyBuilder + auto policy_builder = std::make_unique(); + (*policy_builder) + .AllowFutexOp(FUTEX_WAIT_PRIVATE) + .AllowGetPIDs() + .AllowGetRandom() + .AllowHandleSignals() + .AllowSyscall(__NR_sysinfo) + .AddFile(ssl_certificate) + .AddFile(ssl_key) + .AddFile(ca_certificates); + // Provide the new PolicyBuilder to ModifyPolicy in CurlSandbox + return CurlSapiSandbox::ModifyPolicy(policy_builder.get()); + } + + std::string ssl_certificate; + std::string ssl_key; + std::string ca_certificates; +}; + +int main(int argc, char* argv[]) { + gflags::ParseCommandLineFlags(&argc, &argv, true); + google::InitGoogleLogging(argv[0]); + + absl::Status status; + + // Get input parameters (should be absolute paths) + if (argc != 5) { + LOG(FATAL) << "wrong number of arguments (4 expected)"; + } + std::string ssl_certificate = argv[1]; + std::string ssl_key = argv[2]; + std::string ssl_key_password = argv[3]; + std::string ca_certificates = argv[4]; + + // Initialize sandbox2 and sapi + CurlSapiSandboxEx3 sandbox(ssl_certificate, ssl_key, ca_certificates); + status = sandbox.Init(); + if (!status.ok()) { + LOG(FATAL) << "Couldn't initialize Sandboxed API: " << status; + } + CurlApi api(&sandbox); + + absl::StatusOr curl_code; + + // Initialize curl (CURL_GLOBAL_DEFAULT = 3) + curl_code = api.curl_global_init(3l); + if (!curl_code.ok() || curl_code.value() != CURLE_OK) { + LOG(FATAL) << "curl_global_init failed: " << curl_code.status(); + } + + // Initialize curl easy handle + absl::StatusOr curl_handle = api.curl_easy_init(); + if (!curl_handle.ok()) { + LOG(FATAL) << "curl_easy_init failed: " << curl_handle.status(); + } + sapi::v::RemotePtr curl(curl_handle.value()); + if (!curl.GetValue()) { + LOG(FATAL) << "curl_easy_init failed: curl is NULL"; + } + + // Specify URL to get (using HTTPS) + sapi::v::ConstCStr url("https://example.com"); + curl_code = api.curl_easy_setopt_ptr(&curl, CURLOPT_URL, url.PtrBefore()); + if (!curl_code.ok() || curl_code.value() != CURLE_OK) { + LOG(FATAL) << "curl_easy_setopt_ptr failed: " << curl_code.status(); + } + + // Set the SSL certificate type to "PEM" + sapi::v::ConstCStr ssl_cert_type("PEM"); + curl_code = api.curl_easy_setopt_ptr(&curl, CURLOPT_SSLCERTTYPE, + ssl_cert_type.PtrBefore()); + if (!curl_code.ok() || curl_code.value() != CURLE_OK) { + LOG(FATAL) << "curl_easy_setopt_ptr failed: " << curl_code.status(); + } + + // Set the certificate for client authentication + sapi::v::ConstCStr sapi_ssl_certificate(ssl_certificate.c_str()); + curl_code = api.curl_easy_setopt_ptr(&curl, CURLOPT_SSLCERT, + sapi_ssl_certificate.PtrBefore()); + if (!curl_code.ok() || curl_code.value() != CURLE_OK) { + LOG(FATAL) << "curl_easy_setopt_ptr failed: " << curl_code.status(); + } + + // Set the private key for client authentication + sapi::v::ConstCStr sapi_ssl_key(ssl_key.c_str()); + curl_code = + api.curl_easy_setopt_ptr(&curl, CURLOPT_SSLKEY, sapi_ssl_key.PtrBefore()); + if (!curl_code.ok() || curl_code.value() != CURLE_OK) { + LOG(FATAL) << "curl_easy_setopt_ptr failed: " << curl_code.status(); + } + + // Set the password used to protect the private key + sapi::v::ConstCStr sapi_ssl_key_password(ssl_key_password.c_str()); + curl_code = api.curl_easy_setopt_ptr(&curl, CURLOPT_KEYPASSWD, + sapi_ssl_key_password.PtrBefore()); + if (!curl_code.ok() || curl_code.value() != CURLE_OK) { + LOG(FATAL) << "curl_easy_setopt_ptr failed: " << curl_code.status(); + } + + // Set the file with the certificates vaildating the server + sapi::v::ConstCStr sapi_ca_certificates(ca_certificates.c_str()); + curl_code = api.curl_easy_setopt_ptr(&curl, CURLOPT_CAINFO, + sapi_ca_certificates.PtrBefore()); + if (!curl_code.ok() || curl_code.value() != CURLE_OK) { + LOG(FATAL) << "curl_easy_setopt_ptr failed: " << curl_code.status(); + } + + // Verify the authenticity of the server + curl_code = api.curl_easy_setopt_long(&curl, CURLOPT_SSL_VERIFYPEER, 1L); + if (!curl_code.ok() || curl_code.value() != CURLE_OK) { + LOG(FATAL) << "curl_easy_setopt_long failed: " << curl_code.status(); + } + + // Perform the request + curl_code = api.curl_easy_perform(&curl); + if (!curl_code.ok() || curl_code.value() != CURLE_OK) { + LOG(FATAL) << "curl_easy_perform failed: " << curl_code.status(); + } + + // Cleanup curl easy handle + status = api.curl_easy_cleanup(&curl); + if (!status.ok()) { + LOG(FATAL) << "curl_easy_cleanup failed: " << status; + } + + // Cleanup curl + status = api.curl_global_cleanup(); + if (!status.ok()) { + LOG(FATAL) << "curl_global_cleanup failed: " << status; + } + + return EXIT_SUCCESS; +} diff --git a/oss-internship-2020/curl/examples/example4.cc b/oss-internship-2020/curl/examples/example4.cc new file mode 100644 index 0000000..9103866 --- /dev/null +++ b/oss-internship-2020/curl/examples/example4.cc @@ -0,0 +1,128 @@ +// Copyright 2020 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Sandboxed version of multi-poll.c +// HTTP GET request with polling + +#include + +#include "../sandbox.h" // NOLINT(build/include) +#include "curl_sapi.sapi.h" // NOLINT(build/include) +#include "sandboxed_api/util/flag.h" + +int main(int argc, char* argv[]) { + gflags::ParseCommandLineFlags(&argc, &argv, true); + google::InitGoogleLogging(argv[0]); + + absl::Status status; + + // Initialize sandbox2 and sapi + CurlSapiSandbox sandbox; + status = sandbox.Init(); + if (!status.ok()) { + LOG(FATAL) << "Couldn't initialize Sandboxed API: " << status; + } + CurlApi api(&sandbox); + + // Number of running handles + sapi::v::Int still_running(1); + + absl::StatusOr curl_code; + + // Initialize curl (CURL_GLOBAL_DEFAULT = 3) + curl_code = api.curl_global_init(3l); + if (!curl_code.ok() || curl_code.value() != CURLE_OK) { + LOG(FATAL) << "curl_global_init failed: " << curl_code.status(); + } + + // Initialize http_handle + absl::StatusOr curl_handle = api.curl_easy_init(); + if (!curl_handle.ok()) { + LOG(FATAL) << "curl_easy_init failed: " << curl_handle.status(); + } + sapi::v::RemotePtr http_handle(curl_handle.value()); + if (!http_handle.GetValue()) { + LOG(FATAL) << "curl_easy_init failed: http_handle is NULL"; + } + + // Specify URL to get + sapi::v::ConstCStr url("http://example.com"); + curl_code = + api.curl_easy_setopt_ptr(&http_handle, CURLOPT_URL, url.PtrBefore()); + if (!curl_code.ok() || curl_code.value() != CURLE_OK) { + LOG(FATAL) << "curl_easy_setopt_ptr failed: " << curl_code.status(); + } + + // Initialize multi_handle + absl::StatusOr curlm_handle = api.curl_multi_init(); + if (!curlm_handle.ok()) { + LOG(FATAL) << "curl_multi_init failed: " << curlm_handle.status(); + } + sapi::v::RemotePtr multi_handle(curlm_handle.value()); + if (!multi_handle.GetValue()) { + LOG(FATAL) << "curl_multi_init failed: multi_handle is NULL"; + } + + // Add http_handle to the multi stack + curl_code = api.curl_multi_add_handle(&multi_handle, &http_handle); + if (!curl_code.ok() || curl_code.value() != CURLE_OK) { + LOG(FATAL) << "curl_multi_add_handle failed: " << curl_code.status(); + } + + while (still_running.GetValue()) { + sapi::v::Int numfds(0); + + // Perform the request + curl_code = api.curl_multi_perform(&multi_handle, still_running.PtrBoth()); + if (!curl_code.ok() || curl_code.value() != CURLE_OK) { + LOG(FATAL) << "curl_mutli_perform failed: " << curl_code.status(); + } + + if (still_running.GetValue()) { + // Wait for an event or timeout + sapi::v::NullPtr null_ptr; + curl_code = api.curl_multi_poll_sapi(&multi_handle, &null_ptr, 0, 1000, + numfds.PtrBoth()); + if (!curl_code.ok() || curl_code.value() != CURLE_OK) { + LOG(FATAL) << "curl_multi_poll_sapi failed: " << curl_code.status(); + } + } + } + + // Remove http_handle from the multi stack + curl_code = api.curl_multi_remove_handle(&multi_handle, &http_handle); + if (!curl_code.ok() || curl_code.value() != CURLE_OK) { + LOG(FATAL) << "curl_multi_remove_handle failed: " << curl_code.status(); + } + + // Cleanup http_handle + status = api.curl_easy_cleanup(&http_handle); + if (!status.ok()) { + LOG(FATAL) << "curl_easy_cleanup failed: " << status; + } + + // Cleanup multi_handle + curl_code = api.curl_multi_cleanup(&multi_handle); + if (!curl_code.ok() || curl_code.value() != CURLE_OK) { + LOG(FATAL) << "curl_multi_cleanup failed: " << curl_code.status(); + } + + // Cleanup curl + status = api.curl_global_cleanup(); + if (!status.ok()) { + LOG(FATAL) << "curl_global_cleanup failed: " << status; + } + + return EXIT_SUCCESS; +} diff --git a/oss-internship-2020/curl/examples/example5.cc b/oss-internship-2020/curl/examples/example5.cc new file mode 100644 index 0000000..8cc6a16 --- /dev/null +++ b/oss-internship-2020/curl/examples/example5.cc @@ -0,0 +1,102 @@ +// Copyright 2020 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Sandboxed version of multithread.c +// Multithreaded HTTP GET requests + +#include + +#include + +#include "../sandbox.h" // NOLINT(build/include) + +void pull_one_url(const std::string& url, CurlApi& api) { + // Initialize the curl session + absl::StatusOr curl_handle = api.curl_easy_init(); + if (!curl_handle.ok()) { + LOG(FATAL) << "curl_easy_init failed: " << curl_handle.status(); + } + sapi::v::RemotePtr curl(curl_handle.value()); + if (!curl.GetValue()) { + LOG(FATAL) << "curl_easy_init failed: curl is NULL"; + } + + absl::StatusOr curl_code; + + // Specify URL to get + sapi::v::ConstCStr sapi_url(url.c_str()); + curl_code = + api.curl_easy_setopt_ptr(&curl, CURLOPT_URL, sapi_url.PtrBefore()); + if (!curl_code.ok() || curl_code.value() != CURLE_OK) { + LOG(FATAL) << "curl_easy_setopt_ptr failed: " << curl_code.status(); + } + + // Perform the request + curl_code = api.curl_easy_perform(&curl); + if (!curl_code.ok() || curl_code.value() != CURLE_OK) { + LOG(FATAL) << "curl_easy_perform failed: " << curl_code.status(); + } + + // Cleanup curl + absl::Status status = api.curl_easy_cleanup(&curl); + if (!status.ok()) { + LOG(FATAL) << "curl_easy_cleanup failed: " << status; + } +} + +const std::vector urls = { + "http://example.com", "http://example.edu", "http://example.net", + "http://example.org"}; + +int main(int argc, char* argv[]) { + gflags::ParseCommandLineFlags(&argc, &argv, true); + google::InitGoogleLogging(argv[0]); + + absl::Status status; + + // Initialize sandbox2 and sapi + CurlSapiSandbox sandbox; + status = sandbox.Init(); + if (!status.ok()) { + LOG(FATAL) << "Couldn't initialize Sandboxed API: " << status; + } + CurlApi api(&sandbox); + + absl::StatusOr curl_code; + + // Initialize curl (CURL_GLOBAL_DEFAULT = 3) + curl_code = api.curl_global_init(3l); + if (!curl_code.ok() || curl_code.value() != CURLE_OK) { + LOG(FATAL) << "curl_global_init failed: " << curl_code.status(); + } + + // Create the threads + std::vector threads; + for (auto& url : urls) { + threads.emplace_back(pull_one_url, std::ref(url), std::ref(api)); + } + + // Join the threads + for (auto& thread : threads) { + thread.join(); + } + + // Cleanup curl + status = api.curl_global_cleanup(); + if (!status.ok()) { + LOG(FATAL) << "curl_global_cleanup failed: " << status; + } + + return EXIT_SUCCESS; +} diff --git a/oss-internship-2020/curl/examples/example6.cc b/oss-internship-2020/curl/examples/example6.cc new file mode 100644 index 0000000..15fb1c2 --- /dev/null +++ b/oss-internship-2020/curl/examples/example6.cc @@ -0,0 +1,72 @@ +// Copyright 2020 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Sandboxed version of simple.c using transactions +// Simple HTTP GET request + +#include + +#include "../sandbox.h" // NOLINT(build/include) +#include "sandboxed_api/transaction.h" + +class CurlTransaction : public sapi::Transaction { + public: + explicit CurlTransaction(std::unique_ptr sandbox) + : sapi::Transaction(std::move(sandbox)) { + sapi::Transaction::SetTimeLimit(kTimeOutVal); + } + + private: + // Default timeout value for each transaction run. + static constexpr time_t kTimeOutVal = 2; + + // The main processing function. + absl::Status Main() override; +}; + +absl::Status CurlTransaction::Main() { + CurlApi api(sandbox()); + + // Initialize the curl session + SAPI_ASSIGN_OR_RETURN(void* curl_remote, api.curl_easy_init()); + sapi::v::RemotePtr curl(curl_remote); + TRANSACTION_FAIL_IF_NOT(curl.GetValue(), "curl_easy_init failed"); + + // Specify URL to get + sapi::v::ConstCStr url("http://example.com"); + SAPI_ASSIGN_OR_RETURN( + int setopt_url_code, + api.curl_easy_setopt_ptr(&curl, CURLOPT_URL, url.PtrBefore())); + TRANSACTION_FAIL_IF_NOT(setopt_url_code == CURLE_OK, + "curl_easy_setopt_ptr failed"); + + // Perform the request + SAPI_ASSIGN_OR_RETURN(int perform_code, api.curl_easy_perform(&curl)); + TRANSACTION_FAIL_IF_NOT(setopt_url_code == CURLE_OK, + "curl_easy_perform failed"); + + // Cleanup curl + TRANSACTION_FAIL_IF_NOT(api.curl_easy_cleanup(&curl).ok(), + "curl_easy_cleanup failed"); + + return absl::OkStatus(); +} + +int main(int argc, char* argv[]) { + CurlTransaction curl{std::make_unique()}; + absl::Status status = curl.Run(); + CHECK(status.ok()) << "CurlTransaction failed"; + + return EXIT_SUCCESS; +} diff --git a/oss-internship-2020/curl/sandbox.h b/oss-internship-2020/curl/sandbox.h new file mode 100644 index 0000000..914617f --- /dev/null +++ b/oss-internship-2020/curl/sandbox.h @@ -0,0 +1,67 @@ +// Copyright 2020 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef SANDBOX_H_ +#define SANDBOX_H_ + +#include +#include // For mmap arguments +#include + +#include + +#include "curl_sapi.sapi.h" // NOLINT(build/include) +#include "sandboxed_api/sandbox2/util/bpf_helper.h" + +class CurlSapiSandbox : public CurlSandbox { + protected: + std::unique_ptr ModifyPolicy( + sandbox2::PolicyBuilder* policy_builder) override { + // Return a new policy + return (*policy_builder) + .AllowDynamicStartup() + .AllowExit() + .AllowFork() + .AllowFutexOp(FUTEX_WAKE_PRIVATE) + .AllowMmap() + .AllowOpen() + .AllowRead() + .AllowSafeFcntl() + .AllowWrite() + .AllowSyscalls({ + __NR_accept, + __NR_access, + __NR_bind, + __NR_connect, + __NR_getpeername, + __NR_getsockname, + __NR_getsockopt, + __NR_ioctl, + __NR_listen, + __NR_madvise, + __NR_poll, + __NR_recvfrom, + __NR_recvmsg, + __NR_sendmmsg, + __NR_sendto, + __NR_setsockopt, + __NR_socket, + }) + .AllowUnrestrictedNetworking() + .AddDirectory("/lib") + .BuildOrDie(); + } +}; + +#endif // SANDBOX_H_ diff --git a/oss-internship-2020/curl/tests/CMakeLists.txt b/oss-internship-2020/curl/tests/CMakeLists.txt new file mode 100644 index 0000000..6fd4556 --- /dev/null +++ b/oss-internship-2020/curl/tests/CMakeLists.txt @@ -0,0 +1,28 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +include(GoogleTest) + +add_executable(tests + test_utils.h + test_utils.cc + tests.cc +) + +target_link_libraries(tests + curl_sapi sapi::sapi + gtest gmock gtest_main +) + +gtest_discover_tests(tests) diff --git a/oss-internship-2020/curl/tests/test_utils.cc b/oss-internship-2020/curl/tests/test_utils.cc new file mode 100644 index 0000000..8ea2897 --- /dev/null +++ b/oss-internship-2020/curl/tests/test_utils.cc @@ -0,0 +1,301 @@ +// Copyright 2020 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "test_utils.h" // NOLINT(build/include) + +#include +#include +#include +#include +#include +#include +#include + +#include +#include // NOLINT(build/c++11) + +int CurlTestUtils::port_; +std::thread CurlTestUtils::server_thread_; + +absl::Status CurlTestUtils::CurlTestSetUp() { + // Initialize sandbox2 and sapi + sandbox_ = std::make_unique(); + absl::Status init = sandbox_->Init(); + if (!init.ok()) { + return init; + } + api_ = std::make_unique(sandbox_.get()); + + // Initialize curl + absl::StatusOr curl_handle = api_->curl_easy_init(); + if (!curl_handle.ok()) { + return curl_handle.status(); + } + if (!curl_handle.value()) { + return absl::UnavailableError("curl_easy_init returned NULL "); + } + curl_ = std::make_unique(curl_handle.value()); + + absl::StatusOr curl_code; + + // Specify request URL + sapi::v::ConstCStr sapi_url(kUrl.data()); + curl_code = api_->curl_easy_setopt_ptr(curl_.get(), CURLOPT_URL, + sapi_url.PtrBefore()); + if (!curl_code.ok()) { + return curl_code.status(); + } + if (curl_code.value() != CURLE_OK) { + return absl::UnavailableError( + "curl_easy_setopt_ptr returned with the error code " + + curl_code.value()); + } + + // Set port + curl_code = api_->curl_easy_setopt_long(curl_.get(), CURLOPT_PORT, port_); + if (!curl_code.ok()) { + return curl_code.status(); + } + if (curl_code.value() != CURLE_OK) { + return absl::UnavailableError( + "curl_easy_setopt_long returned with the error code " + + curl_code.value()); + } + + // Generate pointer to the WriteToMemory callback + void* function_ptr; + absl::Status symbol = + sandbox_->rpc_channel()->Symbol("WriteToMemory", &function_ptr); + if (!symbol.ok()) { + return symbol; + } + sapi::v::RemotePtr remote_function_ptr(function_ptr); + + // Set WriteToMemory as the write function + curl_code = api_->curl_easy_setopt_ptr(curl_.get(), CURLOPT_WRITEFUNCTION, + &remote_function_ptr); + if (!curl_code.ok()) { + return curl_code.status(); + } + if (curl_code.value() != CURLE_OK) { + return absl::UnavailableError( + "curl_easy_setopt_ptr returned with the error code " + + curl_code.value()); + } + + // Pass memory chunk object to the callback + chunk_ = std::make_unique(0); + curl_code = api_->curl_easy_setopt_ptr(curl_.get(), CURLOPT_WRITEDATA, + chunk_->PtrBoth()); + if (!curl_code.ok()) { + return curl_code.status(); + } + if (curl_code.value() != CURLE_OK) { + return absl::UnavailableError( + "curl_easy_setopt_ptr returned with the error code " + + curl_code.value()); + } + + return absl::OkStatus(); +} + +absl::Status CurlTestUtils::CurlTestTearDown() { + // Cleanup curl + return api_->curl_easy_cleanup(curl_.get()); +} + +absl::StatusOr CurlTestUtils::PerformRequest() { + // Perform the request + absl::StatusOr curl_code = api_->curl_easy_perform(curl_.get()); + if (!curl_code.ok()) { + return curl_code.status(); + } + if (curl_code.value() != CURLE_OK) { + return absl::UnavailableError( + "curl_easy_perform returned with the error code " + curl_code.value()); + } + + // Get pointer to the memory chunk + absl::Status status = sandbox_->TransferFromSandboxee(chunk_.get()); + if (!status.ok()) { + return status; + } + + return std::string{reinterpret_cast(chunk_->GetData())}; +} + +namespace { + +// Read the socket until str is completely read +std::string ReadUntil(const int socket, const std::string& str, + const size_t max_request_size) { + std::string str_read; + str_read.reserve(max_request_size); + + // Read one char at a time until str is suffix of buf + while (!absl::EndsWith(str_read, str)) { + char next_char; + if (str_read.size() >= max_request_size || + read(socket, &next_char, 1) < 1) { + return ""; + } + str_read += next_char; + } + + return str_read; +} + +// Parse HTTP headers to return the Content-Length +ssize_t GetContentLength(const std::string& headers) { + // Find the Content-Length header + std::string::size_type length_header_start = headers.find("Content-Length: "); + + // There is no Content-Length field + if (length_header_start == std::string::npos) { + return 0; + } + + // Find Content-Length string + std::string::size_type length_start = + length_header_start + std::string{"Content-Length: "}.size(); + std::string::size_type length_bytes = + headers.find("\r\n", length_start) - length_start; + + // length_bytes exceeds maximum + if (length_bytes >= 64) { + return -1; + } + + // Convert string to int and return + return std::stoi(headers.substr(length_start, length_bytes)); +} + +// Read exactly content_bytes from the socket +std::string ReadExact(int socket, size_t content_bytes) { + std::string str_read; + str_read.reserve(content_bytes); + + // Read one char at a time until all chars are read + while (str_read.size() < content_bytes) { + char next_char; + if (read(socket, &next_char, 1) < 1) { + return ""; + } + str_read += next_char; + } + + return str_read; +} + +// Listen on the socket and answer back to requests +void ServerLoop(int listening_socket, sockaddr_in socket_address) { + socklen_t socket_address_size = sizeof(socket_address); + + // Listen on the socket (maximum 1 connection) + if (listen(listening_socket, 1) == -1) { + return; + } + + // Keep accepting connections until the thread is terminated + // (i.e. server_thread_ is assigned to a new thread or destroyed) + for (;;) { + // File descriptor to the connection socket + // This blocks the thread until a connection is established + int accepted_socket = + accept(listening_socket, reinterpret_cast(&socket_address), + reinterpret_cast(&socket_address_size)); + if (accepted_socket == -1) { + return; + } + + constexpr int kMaxRequestSize = 4096; + + // Read until the end of the headers + std::string headers = + ReadUntil(accepted_socket, "\r\n\r\n", kMaxRequestSize); + + if (headers == "") { + close(accepted_socket); + return; + } + + // Get the length of the request content + ssize_t content_length = GetContentLength(headers); + if (content_length > kMaxRequestSize - headers.size() || + content_length < 0) { + close(accepted_socket); + return; + } + + // Read the request content + std::string content = ReadExact(accepted_socket, content_length); + + // Prepare a response for the request + std::string http_response = + "HTTP/1.1 200 OK\nContent-Type: text/plain\nContent-Length: "; + + if (headers.substr(0, 3) == "GET") { + http_response += "2\r\n\r\nOK"; + + } else if (headers.substr(0, 4) == "POST") { + http_response += + std::to_string(content.size()) + "\r\n\r\n" + std::string{content}; + + } else { + close(accepted_socket); + return; + } + + // Ignore any errors, the connection will be closed anyway + write(accepted_socket, http_response.c_str(), http_response.size()); + + // Close the socket + close(accepted_socket); + } +} + +} // namespace + +void CurlTestUtils::StartMockServer() { + // Get the socket file descriptor + int listening_socket = socket(AF_INET, SOCK_STREAM, 0); + + // Create the socket address object + // The port is set to 0, meaning that it will be auto assigned + // Only local connections can access this socket + sockaddr_in socket_address{AF_INET, 0, htonl(INADDR_LOOPBACK)}; + socklen_t socket_address_size = sizeof(socket_address); + if (listening_socket == -1) { + return; + } + + // Bind the file descriptor to the socket address object + if (bind(listening_socket, reinterpret_cast(&socket_address), + socket_address_size) == -1) { + return; + } + + // Assign an available port to the socket address object + if (getsockname(listening_socket, + reinterpret_cast(&socket_address), + &socket_address_size) == -1) { + return; + } + + // Get the port number + port_ = ntohs(socket_address.sin_port); + + // Set server_thread_ operation to socket listening + server_thread_ = std::thread(ServerLoop, listening_socket, socket_address); +} diff --git a/oss-internship-2020/curl/tests/test_utils.h b/oss-internship-2020/curl/tests/test_utils.h new file mode 100644 index 0000000..fa74769 --- /dev/null +++ b/oss-internship-2020/curl/tests/test_utils.h @@ -0,0 +1,55 @@ +// Copyright 2020 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef TESTS_H_ +#define TESTS_H_ + +#include "../sandbox.h" // NOLINT(build/include) +#include "curl_sapi.sapi.h" // NOLINT(build/include) +#include "gtest/gtest.h" +#include "sandboxed_api/util/flag.h" +#include "sandboxed_api/util/status_matchers.h" + +// Helper class that can be used to test Curl Sandboxed +class CurlTestUtils { + protected: + // Initialize and set up the curl handle + absl::Status CurlTestSetUp(); + // Clean up the curl handle + absl::Status CurlTestTearDown(); + + // Perform a request to the mock server, return the response + absl::StatusOr PerformRequest(); + + // Start a mock server (only once) that will manage connections for the tests + // The server listens on a port asynchronously by creating a thread + // The port number is stored in port_ + // Responds with "OK" to a GET request + // Responds with the POST request fields to a POST request + static void StartMockServer(); + + std::unique_ptr sandbox_; + std::unique_ptr api_; + std::unique_ptr curl_; + + static std::thread server_thread_; + + static constexpr absl::string_view kUrl = "http://127.0.0.1/"; + static int port_; + + private: + std::unique_ptr chunk_; +}; + +#endif // TESTS_H_ diff --git a/oss-internship-2020/curl/tests/tests.cc b/oss-internship-2020/curl/tests/tests.cc new file mode 100644 index 0000000..c786169 --- /dev/null +++ b/oss-internship-2020/curl/tests/tests.cc @@ -0,0 +1,138 @@ +// Copyright 2020 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "test_utils.h" // NOLINT(build/include) + +class CurlTest : public CurlTestUtils, public ::testing::Test { + protected: + void SetUp() override { + // Start mock server, get port number and check for any error + StartMockServer(); + ASSERT_TRUE(server_thread_.joinable()); + ASSERT_TRUE(CurlTestSetUp().ok()); + } + + void TearDown() override { + ASSERT_TRUE(CurlTestTearDown().ok()); + // Detach the server thread + server_thread_.detach(); + } +}; + +TEST_F(CurlTest, EffectiveUrl) { + sapi::v::RemotePtr effective_url_ptr(nullptr); + + ASSERT_TRUE(PerformRequest().ok()); + + // Get effective URL + SAPI_ASSERT_OK_AND_ASSIGN( + int getinfo_code, + api_->curl_easy_getinfo_ptr(curl_.get(), CURLINFO_EFFECTIVE_URL, + effective_url_ptr.PtrBoth())); + ASSERT_EQ(getinfo_code, CURLE_OK); + + // Store effective URL in a string + SAPI_ASSERT_OK_AND_ASSIGN(std::string effective_url, + sandbox_->GetCString(sapi::v::RemotePtr( + effective_url_ptr.GetPointedVar()))); + + // Compare effective URL with original URL + ASSERT_EQ(effective_url, kUrl); +} + +TEST_F(CurlTest, EffectivePort) { + sapi::v::Int effective_port; + + ASSERT_TRUE(PerformRequest().ok()); + + // Get effective port + SAPI_ASSERT_OK_AND_ASSIGN(int getinfo_code, api_->curl_easy_getinfo_ptr( + curl_.get(), CURLINFO_PRIMARY_PORT, + effective_port.PtrBoth())); + ASSERT_EQ(getinfo_code, CURLE_OK); + + // Compare effective port with port set by the mock server + ASSERT_EQ(effective_port.GetValue(), port_); +} + +TEST_F(CurlTest, ResponseCode) { + sapi::v::Int response_code; + + ASSERT_TRUE(PerformRequest().ok()); + + // Get response code + SAPI_ASSERT_OK_AND_ASSIGN( + int getinfo_code, + api_->curl_easy_getinfo_ptr(curl_.get(), CURLINFO_RESPONSE_CODE, + response_code.PtrBoth())); + ASSERT_EQ(getinfo_code, CURLE_OK); + + // Check response code + ASSERT_EQ(response_code.GetValue(), 200); +} + +TEST_F(CurlTest, ContentType) { + sapi::v::RemotePtr content_type_ptr(nullptr); + + ASSERT_TRUE(PerformRequest().ok()); + + // Get effective URL + SAPI_ASSERT_OK_AND_ASSIGN(int getinfo_code, api_->curl_easy_getinfo_ptr( + curl_.get(), CURLINFO_CONTENT_TYPE, + content_type_ptr.PtrBoth())); + ASSERT_EQ(getinfo_code, CURLE_OK); + + // Store content type in a string + SAPI_ASSERT_OK_AND_ASSIGN(std::string content_type, + sandbox_->GetCString(sapi::v::RemotePtr( + content_type_ptr.GetPointedVar()))); + + // Compare content type with "text/plain" + ASSERT_EQ(content_type, "text/plain"); +} + +TEST_F(CurlTest, GETResponse) { + SAPI_ASSERT_OK_AND_ASSIGN(std::string response, PerformRequest()); + + // Compare response with expected response + ASSERT_EQ(response, "OK"); +} + +TEST_F(CurlTest, POSTResponse) { + sapi::v::ConstCStr post_fields("postfields"); + + // Set request method to POST + SAPI_ASSERT_OK_AND_ASSIGN(int setopt_post, api_->curl_easy_setopt_long( + curl_.get(), CURLOPT_POST, 1l)); + ASSERT_EQ(setopt_post, CURLE_OK); + + // Set the size of the POST fields + SAPI_ASSERT_OK_AND_ASSIGN( + int setopt_post_fields_size, + api_->curl_easy_setopt_long(curl_.get(), CURLOPT_POSTFIELDSIZE, + post_fields.GetSize())); + ASSERT_EQ(setopt_post_fields_size, CURLE_OK); + + // Set the POST fields + SAPI_ASSERT_OK_AND_ASSIGN( + int setopt_post_fields, + api_->curl_easy_setopt_ptr(curl_.get(), CURLOPT_POSTFIELDS, + post_fields.PtrBefore())); + ASSERT_EQ(setopt_post_fields, CURLE_OK); + + SAPI_ASSERT_OK_AND_ASSIGN(std::string response, PerformRequest()); + + // Compare response with expected response + ASSERT_EQ(std::string(post_fields.GetData()), response); +} diff --git a/oss-internship-2020/guetzli/.bazelrc b/oss-internship-2020/guetzli/.bazelrc new file mode 100644 index 0000000..a68e070 --- /dev/null +++ b/oss-internship-2020/guetzli/.bazelrc @@ -0,0 +1,2 @@ +# Build in C++17 mode without a custom CROSSTOOL +build --cxxopt=-std=c++17 diff --git a/oss-internship-2020/guetzli/BUILD.bazel b/oss-internship-2020/guetzli/BUILD.bazel new file mode 100644 index 0000000..b00b12e --- /dev/null +++ b/oss-internship-2020/guetzli/BUILD.bazel @@ -0,0 +1,83 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +load( + "@com_google_sandboxed_api//sandboxed_api/bazel:sapi.bzl", + "sapi_library", +) + +licenses(["notice"]) + +cc_library( + name = "guetzli_wrapper", + srcs = ["guetzli_entry_points.cc"], + hdrs = ["guetzli_entry_points.h"], + deps = [ + "@com_google_sandboxed_api//sandboxed_api:lenval_core", + "@com_google_sandboxed_api//sandboxed_api:vars", + "@guetzli//:guetzli_lib", + "@png_archive//:png", + ], +) + +sapi_library( + name = "guetzli_sapi", + srcs = ["guetzli_transaction.cc"], + hdrs = [ + "guetzli_sandbox.h", + "guetzli_transaction.h", + ], + functions = [ + "ProcessJpeg", + "ProcessRgb", + "WriteDataToFd", + ], + input_files = ["guetzli_entry_points.h"], + lib = ":guetzli_wrapper", + lib_name = "Guetzli", + namespace = "guetzli::sandbox", + visibility = ["//visibility:public"], +) + +cc_binary( + name = "guetzli_sandboxed", + srcs = ["guetzli_sandboxed.cc"], + deps = [ + ":guetzli_sapi", + ], +) + +cc_test( + name = "transaction_tests", + size = "large", + srcs = ["guetzli_transaction_test.cc"], + data = glob(["testdata/*"]), + visibility = ["//visibility:public"], + deps = [ + "//:guetzli_sapi", + "@com_google_googletest//:gtest_main", + ], +) + +cc_test( + name = "sapi_lib_tests", + size = "large", + srcs = ["guetzli_sapi_test.cc"], + data = glob(["testdata/*"]), + visibility = ["//visibility:public"], + deps = [ + "//:guetzli_sapi", + "@com_google_googletest//:gtest_main", + ], +) diff --git a/oss-internship-2020/guetzli/README.md b/oss-internship-2020/guetzli/README.md new file mode 100644 index 0000000..5fa8412 --- /dev/null +++ b/oss-internship-2020/guetzli/README.md @@ -0,0 +1,32 @@ +# Guetzli Sandbox +This is an example implementation of a sandbox for the [Guetzli](https://github.com/google/guetzli) library using [Sandboxed API](https://github.com/google/sandboxed-api). +Please read Guetzli's [documentation](https://github.com/google/guetzli#introduction) to learn more about it. + +## Implementation details +Because Guetzli provides a C++ API and SAPI requires functions to be `extern "C"`, a wrapper library has been written for the compatibility. SAPI provides a Transaction class, which is a convenient way to create a wrapper for your sandboxed API that handles internal errors. The original Guetzli has a command-line utility to encode images, so a fully compatible utility that uses sandboxed Guetzli is provided. + +The wrapper around Guetzli uses file descriptors to pass data to the sandbox. This approach restricts the sandbox from using the `open()` syscall and also helps to prevent making copies of data, because you need to synchronize it between processes. + +## Build Guetzli Sandboxed +Right now Sandboxed API support only Linux systems, so you need one to build it. Guetzli sandboxed uses [Bazel](https://bazel.build/) as a build system so you need to [install it](https://docs.bazel.build/versions/3.4.0/install.html) before building. + +To build Guetzli sandboxed encoding utility you can use this command: +`bazel build //:guetzli_sandboxed` + +Then you can use it in this way: +``` +guetzli_sandboxed [--quality Q] [--verbose] original.png output.jpg +guetzli_sandboxed [--quality Q] [--verbose] original.jpg output.jpg +``` +Refer to Guetzli's [documentation](https://github.com/google/guetzli#using) to read more about usage. + +## Examples +There are two different sets of unit tests which demonstrate how to use different parts of Guetzli sandboxed: +* `tests/guetzli_sapi_test.cc` - example usage of Guetzli sandboxed API. +* `tests/guetzli_transaction_test.cc` - example usage of Guetzli transaction. + +To run tests use the following command: +`bazel test ...` + +Also, there is an example of custom security policy for your sandbox in +`guetzli_sandbox.h` diff --git a/oss-internship-2020/guetzli/WORKSPACE b/oss-internship-2020/guetzli/WORKSPACE new file mode 100644 index 0000000..33225b7 --- /dev/null +++ b/oss-internship-2020/guetzli/WORKSPACE @@ -0,0 +1,100 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +workspace(name = "guetzli_sandboxed") + +load("@bazel_tools//tools/build_defs/repo:git.bzl", "git_repository") +load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") +load("@bazel_tools//tools/build_defs/repo:utils.bzl", "maybe") + +# Include the Sandboxed API dependency if it does not already exist in this +# project. This ensures that this workspace plays well with other external +# dependencies that might use Sandboxed API. +maybe( + git_repository, + name = "com_google_sandboxed_api", + # This example depends on the latest master. In an embedding project, it + # is advisable to pin Sandboxed API to a specific revision instead. + # commit = "ba47adc21d4c9bc316f3c7c32b0faaef952c111e", # 2020-05-15 + branch = "master", + remote = "https://github.com/google/sandboxed-api.git", +) + +# From here on, Sandboxed API files are available. The statements below setup +# transitive dependencies such as Abseil. Like above, those will only be +# included if they don't already exist in the project. +load( + "@com_google_sandboxed_api//sandboxed_api/bazel:sapi_deps.bzl", + "sapi_deps", +) + +sapi_deps() + +# Need to separately setup Protobuf dependencies in order for the build rules +# to work. +load("@com_google_protobuf//:protobuf_deps.bzl", "protobuf_deps") + +protobuf_deps() + +http_archive( + name = "guetzli", + build_file = "guetzli.BUILD", + sha256 = "39632357e49db83d9560bf0de560ad833352f36d23b109b0e995b01a37bddb57", + strip_prefix = "guetzli-master", + url = "https://github.com/google/guetzli/archive/master.zip", +) + +http_archive( + name = "butteraugli", + build_file = "butteraugli.BUILD", + sha256 = "39632357e49db83d9560bf0de560ad833352f36d23b109b0e995b01a37bddb57", + strip_prefix = "guetzli-master/third_party/butteraugli", + url = "https://github.com/google/guetzli/archive/master.zip", +) + +http_archive( + name = "png_archive", + build_file = "png.BUILD", + sha256 = "a941dc09ca00148fe7aaf4ecdd6a67579c293678ed1e1cf633b5ffc02f4f8cf7", + strip_prefix = "libpng-1.2.57", + url = "http://github.com/glennrp/libpng/archive/v1.2.57.zip", +) + +http_archive( + name = "jpeg_archive", + build_file = "jpeg.BUILD", + sha256 = "240fd398da741669bf3c90366f58452ea59041cacc741a489b99f2f6a0bad052", + strip_prefix = "jpeg-9b", + url = "http://www.ijg.org/files/jpegsrc.v9b.tar.gz", +) + +http_archive( + name = "net_zlib", + build_file = "zlib.BUILD", + sha256 = "c3e5e9fdd5004dcb542feda5ee4f0ff0744628baf8ed2dd5d66f8ca1197cb1a1", # 2020-04-23 + strip_prefix = "zlib-1.2.11", + urls = [ + "https://mirror.bazel.build/zlib.net/zlib-1.2.11.tar.gz", + "https://www.zlib.net/zlib-1.2.11.tar.gz", + ], +) + +# GoogleTest/GoogleMock +maybe( + http_archive, + name = "com_google_googletest", + sha256 = "a6ab7c7d6fd4dd727f6012b5d85d71a73d3aa1274f529ecd4ad84eb9ec4ff767", # 2020-04-16 + strip_prefix = "googletest-dcc92d0ab6c4ce022162a23566d44f673251eee4", + urls = ["https://github.com/google/googletest/archive/dcc92d0ab6c4ce022162a23566d44f673251eee4.zip"], +) diff --git a/oss-internship-2020/guetzli/external/butteraugli.BUILD b/oss-internship-2020/guetzli/external/butteraugli.BUILD new file mode 100644 index 0000000..d61eb35 --- /dev/null +++ b/oss-internship-2020/guetzli/external/butteraugli.BUILD @@ -0,0 +1,28 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +licenses(["notice"]) + +cc_library( + name = "butteraugli", + srcs = [ + "butteraugli/butteraugli.cc", + "butteraugli/butteraugli.h", + ], + hdrs = [ + "butteraugli/butteraugli.h", + ], + copts = ["-Wno-sign-compare"], + visibility = ["//visibility:public"], +) diff --git a/oss-internship-2020/guetzli/external/guetzli.BUILD b/oss-internship-2020/guetzli/external/guetzli.BUILD new file mode 100644 index 0000000..454ec62 --- /dev/null +++ b/oss-internship-2020/guetzli/external/guetzli.BUILD @@ -0,0 +1,32 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +licenses(["notice"]) + +cc_library( + name = "guetzli_lib", + srcs = glob( + [ + "guetzli/*.h", + "guetzli/*.cc", + "guetzli/*.inc", + ], + exclude = ["guetzli/guetzli.cc"], + ), + copts = [ "-Wno-sign-compare" ], + visibility= [ "//visibility:public" ], + deps = [ + "@butteraugli//:butteraugli", + ], +) diff --git a/oss-internship-2020/guetzli/external/jpeg.BUILD b/oss-internship-2020/guetzli/external/jpeg.BUILD new file mode 100644 index 0000000..92c9ddc --- /dev/null +++ b/oss-internship-2020/guetzli/external/jpeg.BUILD @@ -0,0 +1,89 @@ +# Description: +# The Independent JPEG Group's JPEG runtime library. + +licenses(["notice"]) # custom notice-style license, see LICENSE + +cc_library( + name = "jpeg", + srcs = [ + "cderror.h", + "cdjpeg.h", + "jaricom.c", + "jcapimin.c", + "jcapistd.c", + "jcarith.c", + "jccoefct.c", + "jccolor.c", + "jcdctmgr.c", + "jchuff.c", + "jcinit.c", + "jcmainct.c", + "jcmarker.c", + "jcmaster.c", + "jcomapi.c", + "jconfig.h", + "jcparam.c", + "jcprepct.c", + "jcsample.c", + "jctrans.c", + "jdapimin.c", + "jdapistd.c", + "jdarith.c", + "jdatadst.c", + "jdatasrc.c", + "jdcoefct.c", + "jdcolor.c", + "jdct.h", + "jddctmgr.c", + "jdhuff.c", + "jdinput.c", + "jdmainct.c", + "jdmarker.c", + "jdmaster.c", + "jdmerge.c", + "jdpostct.c", + "jdsample.c", + "jdtrans.c", + "jerror.c", + "jfdctflt.c", + "jfdctfst.c", + "jfdctint.c", + "jidctflt.c", + "jidctfst.c", + "jidctint.c", + "jinclude.h", + "jmemmgr.c", + "jmemnobs.c", + "jmemsys.h", + "jmorecfg.h", + "jquant1.c", + "jquant2.c", + "jutils.c", + "jversion.h", + "transupp.h", + ], + hdrs = [ + "jerror.h", + "jpegint.h", + "jpeglib.h", + ], + includes = ["."], + visibility = ["//visibility:public"], +) + +genrule( + name = "configure", + outs = ["jconfig.h"], + cmd = "cat <$@\n" + + "#define HAVE_PROTOTYPES 1\n" + + "#define HAVE_UNSIGNED_CHAR 1\n" + + "#define HAVE_UNSIGNED_SHORT 1\n" + + "#define HAVE_STDDEF_H 1\n" + + "#define HAVE_STDLIB_H 1\n" + + "#ifdef WIN32\n" + + "#define INLINE __inline\n" + + "#else\n" + + "#define INLINE __inline__\n" + + "#endif\n" + + "EOF\n", +) diff --git a/oss-internship-2020/guetzli/external/png.BUILD b/oss-internship-2020/guetzli/external/png.BUILD new file mode 100644 index 0000000..4b061f5 --- /dev/null +++ b/oss-internship-2020/guetzli/external/png.BUILD @@ -0,0 +1,33 @@ +# Description: +# libpng is the official PNG reference library. + +licenses(["notice"]) # BSD/MIT-like license + +cc_library( + name = "png", + srcs = [ + "png.c", + "pngerror.c", + "pngget.c", + "pngmem.c", + "pngpread.c", + "pngread.c", + "pngrio.c", + "pngrtran.c", + "pngrutil.c", + "pngset.c", + "pngtrans.c", + "pngwio.c", + "pngwrite.c", + "pngwtran.c", + "pngwutil.c", + ], + hdrs = [ + "png.h", + "pngconf.h", + ], + includes = ["."], + linkopts = ["-lm"], + visibility = ["//visibility:public"], + deps = ["@net_zlib//:zlib"], +) diff --git a/oss-internship-2020/guetzli/external/zlib.BUILD b/oss-internship-2020/guetzli/external/zlib.BUILD new file mode 100644 index 0000000..ec4fff0 --- /dev/null +++ b/oss-internship-2020/guetzli/external/zlib.BUILD @@ -0,0 +1,56 @@ +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +licenses(["notice"]) + +cc_library( + name = "zlib", + srcs = [ + "adler32.c", + "compress.c", + "crc32.c", + "deflate.c", + "gzclose.c", + "gzlib.c", + "gzread.c", + "gzwrite.c", + "infback.c", + "inffast.c", + "inflate.c", + "inflate.h", + "inftrees.c", + "trees.c", + "trees.h", + "uncompr.c", + "zutil.c", + ], + hdrs = ["zlib.h"], + # Use -Dverbose=-1 to turn off zlib's trace logging. (#3280) + copts = [ + "-w", + "-Dverbose=-1", + ], + includes = ["."], + textual_hdrs = [ + "crc32.h", + "deflate.h", + "gzguts.h", + "inffast.h", + "inffixed.h", + "inftrees.h", + "zconf.h", + "zutil.h", + ], + visibility = ["//visibility:public"], +) diff --git a/oss-internship-2020/guetzli/guetzli_entry_points.cc b/oss-internship-2020/guetzli/guetzli_entry_points.cc new file mode 100644 index 0000000..8de94b6 --- /dev/null +++ b/oss-internship-2020/guetzli/guetzli_entry_points.cc @@ -0,0 +1,293 @@ +// Copyright 2020 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "guetzli_entry_points.h" // NOLINT(build/include) + +#include + +#include +#include +#include +#include +#include + +#include "guetzli/jpeg_data_reader.h" +#include "guetzli/quality.h" +#include "png.h" // NOLINT(build/include) +#include "sandboxed_api/sandbox2/util/fileops.h" +#include "sandboxed_api/util/statusor.h" + +namespace { + +constexpr int kBytesPerPixel = 350; +constexpr int kLowestMemusageMB = 100; + +struct GuetzliInitData { + std::string in_data; + guetzli::Params params; + guetzli::ProcessStats stats; +}; + +struct ImageData { + int xsize; + int ysize; + std::vector rgb; +}; + +sapi::LenValStruct CreateLenValFromData(const void* data, size_t size) { + void* new_data = malloc(size); + memcpy(new_data, data, size); + return {size, new_data}; +} + +sapi::StatusOr ReadFromFd(int fd) { + struct stat file_data; + int status = fstat(fd, &file_data); + + if (status < 0) { + return absl::FailedPreconditionError("Error reading input from fd"); + } + + std::string result; + result.resize(file_data.st_size); + status = read(fd, result.data(), result.size()); + + if (status < 0) { + return absl::FailedPreconditionError("Error reading input from fd"); + } + + return result; +} + +sapi::StatusOr PrepareDataForProcessing( + const ProcessingParams& processing_params) { + sapi::StatusOr input = ReadFromFd(processing_params.remote_fd); + + if (!input.ok()) { + return input.status(); + } + + guetzli::Params guetzli_params; + guetzli_params.butteraugli_target = static_cast( + guetzli::ButteraugliScoreForQuality(processing_params.quality)); + + guetzli::ProcessStats stats; + + if (processing_params.verbose) { + stats.debug_output_file = stderr; + } + + return GuetzliInitData{std::move(input.value()), guetzli_params, stats}; +} + +inline uint8_t BlendOnBlack(const uint8_t val, const uint8_t alpha) { + return (static_cast(val) * static_cast(alpha) + 128) / 255; +} + +// Modified version of ReadPNG from original guetzli.cc +sapi::StatusOr ReadPNG(const std::string& data) { + std::vector rgb; + int xsize, ysize; + png_structp png_ptr = + png_create_read_struct(PNG_LIBPNG_VER_STRING, nullptr, nullptr, nullptr); + if (!png_ptr) { + return absl::FailedPreconditionError( + "Error reading PNG data from input file"); + } + + png_infop info_ptr = png_create_info_struct(png_ptr); + if (!info_ptr) { + png_destroy_read_struct(&png_ptr, nullptr, nullptr); + return absl::FailedPreconditionError( + "Error reading PNG data from input file"); + } + + if (setjmp(png_jmpbuf(png_ptr)) != 0) { + // Ok we are here because of the setjmp. + png_destroy_read_struct(&png_ptr, &info_ptr, nullptr); + return absl::FailedPreconditionError( + "Error reading PNG data from input file"); + } + + std::istringstream memstream(data, std::ios::in | std::ios::binary); + png_set_read_fn( + png_ptr, static_cast(&memstream), + [](png_structp png_ptr, png_bytep outBytes, png_size_t byteCountToRead) { + std::istringstream& memstream = + *static_cast(png_get_io_ptr(png_ptr)); + + memstream.read(reinterpret_cast(outBytes), byteCountToRead); + + if (memstream.eof()) png_error(png_ptr, "unexpected end of data"); + if (memstream.fail()) png_error(png_ptr, "read from memory error"); + }); + + // The png_transforms flags are as follows: + // packing == convert 1,2,4 bit images, + // strip == 16 -> 8 bits / channel, + // shift == use sBIT dynamics, and + // expand == palettes -> rgb, grayscale -> 8 bit images, tRNS -> alpha. + const unsigned int png_transforms = + PNG_TRANSFORM_PACKING | PNG_TRANSFORM_EXPAND | PNG_TRANSFORM_STRIP_16; + + png_read_png(png_ptr, info_ptr, png_transforms, nullptr); + + png_bytep* row_pointers = png_get_rows(png_ptr, info_ptr); + + xsize = png_get_image_width(png_ptr, info_ptr); + ysize = png_get_image_height(png_ptr, info_ptr); + rgb.resize(3 * xsize * ysize); + + const int components = png_get_channels(png_ptr, info_ptr); + switch (components) { + case 1: { + // GRAYSCALE + for (int y = 0; y < ysize; ++y) { + const uint8_t* row_in = row_pointers[y]; + uint8_t* row_out = &rgb[3 * y * xsize]; + for (int x = 0; x < xsize; ++x) { + const uint8_t gray = row_in[x]; + row_out[3 * x + 0] = gray; + row_out[3 * x + 1] = gray; + row_out[3 * x + 2] = gray; + } + } + break; + } + case 2: { + // GRAYSCALE + ALPHA + for (int y = 0; y < ysize; ++y) { + const uint8_t* row_in = row_pointers[y]; + uint8_t* row_out = &rgb[3 * y * xsize]; + for (int x = 0; x < xsize; ++x) { + const uint8_t gray = BlendOnBlack(row_in[2 * x], row_in[2 * x + 1]); + row_out[3 * x + 0] = gray; + row_out[3 * x + 1] = gray; + row_out[3 * x + 2] = gray; + } + } + break; + } + case 3: { + // RGB + for (int y = 0; y < ysize; ++y) { + const uint8_t* row_in = row_pointers[y]; + uint8_t* row_out = &rgb[3 * y * xsize]; + memcpy(row_out, row_in, 3 * xsize); + } + break; + } + case 4: { + // RGBA + for (int y = 0; y < ysize; ++y) { + const uint8_t* row_in = row_pointers[y]; + uint8_t* row_out = &rgb[3 * y * xsize]; + for (int x = 0; x < xsize; ++x) { + const uint8_t alpha = row_in[4 * x + 3]; + row_out[3 * x + 0] = BlendOnBlack(row_in[4 * x + 0], alpha); + row_out[3 * x + 1] = BlendOnBlack(row_in[4 * x + 1], alpha); + row_out[3 * x + 2] = BlendOnBlack(row_in[4 * x + 2], alpha); + } + } + break; + } + default: + png_destroy_read_struct(&png_ptr, &info_ptr, nullptr); + return absl::FailedPreconditionError( + "Error reading PNG data from input file"); + } + png_destroy_read_struct(&png_ptr, &info_ptr, nullptr); + + return ImageData{xsize, ysize, std::move(rgb)}; +} + +bool CheckMemoryLimitExceeded(int memlimit_mb, int xsize, int ysize) { + double pixels = static_cast(xsize) * ysize; + return memlimit_mb != -1 && + (pixels * kBytesPerPixel / (1 << 20) > memlimit_mb || + memlimit_mb < kLowestMemusageMB); +} + +} // namespace + +extern "C" bool ProcessJpeg(const ProcessingParams* processing_params, + sapi::LenValStruct* output) { + auto processing_data = PrepareDataForProcessing(*processing_params); + + if (!processing_data.ok()) { + std::cerr << processing_data.status().ToString() << std::endl; + return false; + } + + guetzli::JPEGData jpg_header; + if (!guetzli::ReadJpeg(processing_data->in_data, guetzli::JPEG_READ_HEADER, + &jpg_header)) { + std::cerr << "Error reading JPG data from input file" << std::endl; + return false; + } + + if (CheckMemoryLimitExceeded(processing_params->memlimit_mb, jpg_header.width, + jpg_header.height)) { + std::cerr << "Memory limit would be exceeded" << std::endl; + return false; + } + + std::string out_data; + if (!guetzli::Process(processing_data->params, &processing_data->stats, + processing_data->in_data, &out_data)) { + std::cerr << "Guezli processing failed" << std::endl; + return false; + } + + *output = CreateLenValFromData(out_data.data(), out_data.size()); + return true; +} + +extern "C" bool ProcessRgb(const ProcessingParams* processing_params, + sapi::LenValStruct* output) { + auto processing_data = PrepareDataForProcessing(*processing_params); + + if (!processing_data.ok()) { + std::cerr << processing_data.status().ToString() << std::endl; + return false; + } + + auto png_data = ReadPNG(processing_data->in_data); + if (!png_data.ok()) { + std::cerr << "Error reading PNG data from input file" << std::endl; + return false; + } + + if (CheckMemoryLimitExceeded(processing_params->memlimit_mb, png_data->xsize, + png_data->ysize)) { + std::cerr << "Memory limit would be exceeded" << std::endl; + return false; + } + + std::string out_data; + if (!guetzli::Process(processing_data->params, &processing_data->stats, + png_data->rgb, png_data->xsize, png_data->ysize, + &out_data)) { + std::cerr << "Guetzli processing failed" << std::endl; + return false; + } + + *output = CreateLenValFromData(out_data.data(), out_data.size()); + return true; +} + +extern "C" bool WriteDataToFd(int fd, sapi::LenValStruct* data) { + return sandbox2::file_util::fileops::WriteToFD( + fd, static_cast(data->data), data->size); +} diff --git a/oss-internship-2020/guetzli/guetzli_entry_points.h b/oss-internship-2020/guetzli/guetzli_entry_points.h new file mode 100644 index 0000000..0f5eab1 --- /dev/null +++ b/oss-internship-2020/guetzli/guetzli_entry_points.h @@ -0,0 +1,35 @@ +// Copyright 2020 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef GUETZLI_SANDBOXED_GUETZLI_ENTRY_POINTS_H_ +#define GUETZLI_SANDBOXED_GUETZLI_ENTRY_POINTS_H_ + +#include "guetzli/processor.h" +#include "sandboxed_api/lenval_core.h" +#include "sandboxed_api/vars.h" + +struct ProcessingParams { + int remote_fd = -1; + int verbose = 0; + int quality = 0; + int memlimit_mb = 0; +}; + +extern "C" bool ProcessJpeg(const ProcessingParams* processing_params, + sapi::LenValStruct* output); +extern "C" bool ProcessRgb(const ProcessingParams* processing_params, + sapi::LenValStruct* output); +extern "C" bool WriteDataToFd(int fd, sapi::LenValStruct* data); + +#endif // GUETZLI_SANDBOXED_GUETZLI_ENTRY_POINTS_H_ diff --git a/oss-internship-2020/guetzli/guetzli_sandbox.h b/oss-internship-2020/guetzli/guetzli_sandbox.h new file mode 100644 index 0000000..44f9b27 --- /dev/null +++ b/oss-internship-2020/guetzli/guetzli_sandbox.h @@ -0,0 +1,45 @@ +// Copyright 2020 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef GUETZLI_SANDBOXED_GUETZLI_SANDBOX_H_ +#define GUETZLI_SANDBOXED_GUETZLI_SANDBOX_H_ + +#include + +#include "guetzli_sapi.sapi.h" // NOLINT(build/include) + +namespace guetzli::sandbox { + +class GuetzliSapiSandbox : public GuetzliSandbox { + public: + std::unique_ptr ModifyPolicy( + sandbox2::PolicyBuilder*) override { + return sandbox2::PolicyBuilder() + .AllowStaticStartup() + .AllowRead() + .AllowSystemMalloc() + .AllowWrite() + .AllowExit() + .AllowStat() + .AllowSyscalls({ + __NR_futex, __NR_close, + __NR_recvmsg // To work with remote fd + }) + .BuildOrDie(); + } +}; + +} // namespace guetzli::sandbox + +#endif // GUETZLI_SANDBOXED_GUETZLI_SANDBOX_H_ diff --git a/oss-internship-2020/guetzli/guetzli_sandboxed.cc b/oss-internship-2020/guetzli/guetzli_sandboxed.cc new file mode 100644 index 0000000..944a572 --- /dev/null +++ b/oss-internship-2020/guetzli/guetzli_sandboxed.cc @@ -0,0 +1,96 @@ +// Copyright 2020 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include +#include + +#include "guetzli_transaction.h" // NOLINT(build/include) +#include "sandboxed_api/sandbox2/util/fileops.h" +#include "sandboxed_api/util/statusor.h" + +namespace { + +constexpr int kDefaultJPEGQuality = 95; +constexpr int kDefaultMemlimitMB = 6000; + +void Usage() { + fprintf(stderr, + "Guetzli JPEG compressor. Usage: \n" + "guetzli [flags] input_filename output_filename\n" + "\n" + "Flags:\n" + " --verbose - Print a verbose trace of all attempts to standard " + "output.\n" + " --quality Q - Visual quality to aim for, expressed as a JPEG " + "quality value.\n" + " Default value is %d.\n" + " --memlimit M - Memory limit in MB. Guetzli will fail if unable to " + "stay under\n" + " the limit. Default limit is %d MB.\n" + " --nomemlimit - Do not limit memory usage.\n", + kDefaultJPEGQuality, kDefaultMemlimitMB); + exit(1); +} + +} // namespace + +int main(int argc, const char** argv) { + int verbose = 0; + int quality = kDefaultJPEGQuality; + int memlimit_mb = kDefaultMemlimitMB; + + int opt_idx = 1; + for (; opt_idx < argc; opt_idx++) { + if (strnlen(argv[opt_idx], 2) < 2 || argv[opt_idx][0] != '-' || + argv[opt_idx][1] != '-') + break; + + if (!strcmp(argv[opt_idx], "--verbose")) { + verbose = 1; + } else if (!strcmp(argv[opt_idx], "--quality")) { + opt_idx++; + if (opt_idx >= argc) Usage(); + quality = atoi(argv[opt_idx]); // NOLINT(runtime/deprecated_fn) + } else if (!strcmp(argv[opt_idx], "--memlimit")) { + opt_idx++; + if (opt_idx >= argc) Usage(); + memlimit_mb = atoi(argv[opt_idx]); // NOLINT(runtime/deprecated_fn) + } else if (!strcmp(argv[opt_idx], "--nomemlimit")) { + memlimit_mb = -1; + } else if (!strcmp(argv[opt_idx], "--")) { + opt_idx++; + break; + } else { + fprintf(stderr, "Unknown commandline flag: %s\n", argv[opt_idx]); + Usage(); + } + } + + if (argc - opt_idx != 2) { + Usage(); + } + + guetzli::sandbox::TransactionParams params = { + argv[opt_idx], argv[opt_idx + 1], verbose, quality, memlimit_mb}; + + guetzli::sandbox::GuetzliTransaction transaction(std::move(params)); + auto result = transaction.Run(); + + if (!result.ok()) { + std::cerr << result.ToString() << std::endl; + return EXIT_FAILURE; + } + + return EXIT_SUCCESS; +} diff --git a/oss-internship-2020/guetzli/guetzli_sapi_test.cc b/oss-internship-2020/guetzli/guetzli_sapi_test.cc new file mode 100644 index 0000000..cd8bc91 --- /dev/null +++ b/oss-internship-2020/guetzli/guetzli_sapi_test.cc @@ -0,0 +1,124 @@ +// Copyright 2020 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include +#include +#include +#include + +#include +#include +#include +#include + +#include "guetzli_sandbox.h" // NOLINT(build/include) +#include "gtest/gtest.h" +#include "sandboxed_api/sandbox2/util/fileops.h" +#include "sandboxed_api/vars.h" + +namespace guetzli::sandbox::tests { + +namespace { + +constexpr absl::string_view kInPngFilename = "bees.png"; +constexpr absl::string_view kInJpegFilename = "nature.jpg"; +constexpr absl::string_view kPngReferenceFilename = "bees_reference.jpg"; +constexpr absl::string_view kJpegReferenceFIlename = "nature_reference.jpg"; + +constexpr int kDefaultQualityTarget = 95; +constexpr int kDefaultMemlimitMb = 6000; + +constexpr absl::string_view kRelativePathToTestdata = + "/guetzli_sandboxed/testdata/"; + +std::string GetPathToInputFile(absl::string_view filename) { + return absl::StrCat(getenv("TEST_SRCDIR"), kRelativePathToTestdata, filename); +} + +std::string ReadFromFile(const std::string& filename) { + std::ifstream stream(filename, std::ios::binary); + + if (!stream.is_open()) { + return ""; + } + + std::stringstream result; + result << stream.rdbuf(); + return result.str(); +} + +} // namespace + +class GuetzliSapiTest : public ::testing::Test { + protected: + void SetUp() override { + sandbox_ = std::make_unique(); + ASSERT_EQ(sandbox_->Init(), absl::OkStatus()); + api_ = std::make_unique(sandbox_.get()); + } + + std::unique_ptr sandbox_; + std::unique_ptr api_; +}; + +// This test can take up to few minutes depending on your hardware +TEST_F(GuetzliSapiTest, ProcessRGB) { + sapi::v::Fd in_fd(open(GetPathToInputFile(kInPngFilename).c_str(), O_RDONLY)); + ASSERT_TRUE(in_fd.GetValue() != -1) << "Error opening input file"; + ASSERT_EQ(api_->sandbox()->TransferToSandboxee(&in_fd), absl::OkStatus()) + << "Error transfering fd to sandbox"; + ASSERT_TRUE(in_fd.GetRemoteFd() != -1) << "Error opening remote fd"; + sapi::v::Struct processing_params; + *processing_params.mutable_data() = { + in_fd.GetRemoteFd(), 0, kDefaultQualityTarget, kDefaultMemlimitMb}; + sapi::v::LenVal output(0); + sapi::StatusOr processing_result = + api_->ProcessRgb(processing_params.PtrBefore(), output.PtrBoth()); + ASSERT_TRUE(processing_result.value_or(false)) << "Error processing rgb data"; + std::string reference_data = + ReadFromFile(GetPathToInputFile(kPngReferenceFilename)); + ASSERT_EQ(output.GetDataSize(), reference_data.size()) + << "Incorrect result data size"; + ASSERT_EQ( + std::string(output.GetData(), output.GetData() + output.GetDataSize()), + reference_data) + << "Processed data doesn't match reference output"; +} + +// This test can take up to few minutes depending on your hardware +TEST_F(GuetzliSapiTest, ProcessJpeg) { + sapi::v::Fd in_fd( + open(GetPathToInputFile(kInJpegFilename).c_str(), O_RDONLY)); + ASSERT_TRUE(in_fd.GetValue() != -1) << "Error opening input file"; + ASSERT_EQ(api_->sandbox()->TransferToSandboxee(&in_fd), absl::OkStatus()) + << "Error transfering fd to sandbox"; + ASSERT_TRUE(in_fd.GetRemoteFd() != -1) << "Error opening remote fd"; + sapi::v::Struct processing_params; + *processing_params.mutable_data() = { + in_fd.GetRemoteFd(), 0, kDefaultQualityTarget, kDefaultMemlimitMb}; + sapi::v::LenVal output(0); + sapi::StatusOr processing_result = + api_->ProcessJpeg(processing_params.PtrBefore(), output.PtrBoth()); + ASSERT_TRUE(processing_result.value_or(false)) << "Error processing jpg data"; + std::string reference_data = + ReadFromFile(GetPathToInputFile(kJpegReferenceFIlename)); + ASSERT_EQ(output.GetDataSize(), reference_data.size()) + << "Incorrect result data size"; + ASSERT_EQ( + std::string(output.GetData(), output.GetData() + output.GetDataSize()), + reference_data) + << "Processed data doesn't match reference output"; +} + +} // namespace guetzli::sandbox::tests diff --git a/oss-internship-2020/guetzli/guetzli_transaction.cc b/oss-internship-2020/guetzli/guetzli_transaction.cc new file mode 100644 index 0000000..2bd6e8a --- /dev/null +++ b/oss-internship-2020/guetzli/guetzli_transaction.cc @@ -0,0 +1,123 @@ +// Copyright 2020 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "guetzli_transaction.h" // NOLINT(build/include) + +#include +#include +#include +#include + +#include +#include + +namespace guetzli::sandbox { + +absl::Status GuetzliTransaction::Main() { + sapi::v::Fd in_fd(open(params_.in_file, O_RDONLY)); + + if (in_fd.GetValue() < 0) { + return absl::FailedPreconditionError("Error opening input file"); + } + + SAPI_ASSIGN_OR_RETURN(image_type_, GetImageTypeFromFd(in_fd.GetValue())); + SAPI_RETURN_IF_ERROR(sandbox()->TransferToSandboxee(&in_fd)); + + if (in_fd.GetRemoteFd() < 0) { + return absl::FailedPreconditionError( + "Error receiving remote FD: remote input fd is set to -1"); + } + + GuetzliApi api(sandbox()); + sapi::v::LenVal output(0); + + sapi::v::Struct processing_params; + *processing_params.mutable_data() = {in_fd.GetRemoteFd(), params_.verbose, + params_.quality, params_.memlimit_mb}; + + auto result = + image_type_ == ImageType::kJpeg + ? api.ProcessJpeg(processing_params.PtrBefore(), output.PtrBefore()) + : api.ProcessRgb(processing_params.PtrBefore(), output.PtrBefore()); + + if (!result.value_or(false)) { + return absl::FailedPreconditionError(absl::StrCat( + "Error processing ", (image_type_ == ImageType::kJpeg ? "jpeg" : "rgb"), + " data")); + } + + sapi::v::Fd out_fd(open(".", O_TMPFILE | O_RDWR, S_IRUSR | S_IWUSR)); + if (out_fd.GetValue() < 0) { + return absl::FailedPreconditionError("Error creating temp output file"); + } + + SAPI_RETURN_IF_ERROR(sandbox()->TransferToSandboxee(&out_fd)); + + if (out_fd.GetRemoteFd() < 0) { + return absl::FailedPreconditionError( + "Error receiving remote FD: remote output fd is set to -1"); + } + + auto write_result = api.WriteDataToFd(out_fd.GetRemoteFd(), output.PtrNone()); + + if (!write_result.value_or(false)) { + return absl::FailedPreconditionError("Error writing file inside sandbox"); + } + + SAPI_RETURN_IF_ERROR(LinkOutFile(out_fd.GetValue())); + + return absl::OkStatus(); +} + +absl::Status GuetzliTransaction::LinkOutFile(int out_fd) const { + if (access(params_.out_file, F_OK) != -1) { + if (remove(params_.out_file) < 0) { + return absl::FailedPreconditionError(absl::StrCat( + "Error deleting existing output file: ", params_.out_file)); + } + } + + std::string path = absl::StrCat("/proc/self/fd/", out_fd); + + if (linkat(AT_FDCWD, path.c_str(), AT_FDCWD, params_.out_file, + AT_SYMLINK_FOLLOW) < 0) { + return absl::FailedPreconditionError( + absl::StrCat("Error linking: ", params_.out_file)); + } + + return absl::OkStatus(); +} + +sapi::StatusOr GuetzliTransaction::GetImageTypeFromFd(int fd) const { + static const unsigned char kPNGMagicBytes[] = { + 0x89, 'P', 'N', 'G', '\r', '\n', 0x1a, '\n', + }; + char read_buf[sizeof(kPNGMagicBytes)]; + + if (read(fd, read_buf, sizeof(kPNGMagicBytes)) != sizeof(kPNGMagicBytes)) { + return absl::FailedPreconditionError( + "Error determining type of the input file"); + } + + if (lseek(fd, 0, SEEK_SET) != 0) { + return absl::FailedPreconditionError( + "Error returnig cursor to the beginning"); + } + + return memcmp(read_buf, kPNGMagicBytes, sizeof(kPNGMagicBytes)) == 0 + ? ImageType::kPng + : ImageType::kJpeg; +} + +} // namespace guetzli::sandbox diff --git a/oss-internship-2020/guetzli/guetzli_transaction.h b/oss-internship-2020/guetzli/guetzli_transaction.h new file mode 100644 index 0000000..2996908 --- /dev/null +++ b/oss-internship-2020/guetzli/guetzli_transaction.h @@ -0,0 +1,59 @@ +// Copyright 2020 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef GUETZLI_SANDBOXED_GUETZLI_TRANSACTION_H_ +#define GUETZLI_SANDBOXED_GUETZLI_TRANSACTION_H_ + +#include + +#include "guetzli_sandbox.h" // NOLINT(build/include) +#include "sandboxed_api/transaction.h" +#include "sandboxed_api/vars.h" + +namespace guetzli::sandbox { + +enum class ImageType { kJpeg, kPng }; + +struct TransactionParams { + const char* in_file = nullptr; + const char* out_file = nullptr; + int verbose = 0; + int quality = 0; + int memlimit_mb = 0; +}; + +// Instance of this transaction shouldn't be reused +// Create a new one for each processing operation +class GuetzliTransaction : public sapi::Transaction { + public: + explicit GuetzliTransaction(TransactionParams params, int retry_count = 0) + : sapi::Transaction(std::make_unique()), + params_(std::move(params)) { + set_retry_count(retry_count); + SetTimeLimit(absl::InfiniteDuration()); + } + + private: + absl::Status Main() final; + + absl::Status LinkOutFile(int out_fd) const; + sapi::StatusOr GetImageTypeFromFd(int fd) const; + + const TransactionParams params_; + ImageType image_type_ = ImageType::kJpeg; +}; + +} // namespace guetzli::sandbox + +#endif // GUETZLI_SANDBOXED_GUETZLI_TRANSACTION_H_ diff --git a/oss-internship-2020/guetzli/guetzli_transaction_test.cc b/oss-internship-2020/guetzli/guetzli_transaction_test.cc new file mode 100644 index 0000000..c650819 --- /dev/null +++ b/oss-internship-2020/guetzli/guetzli_transaction_test.cc @@ -0,0 +1,145 @@ +// Copyright 2020 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "guetzli_transaction.h" // NOLINT(build/include) + +#include +#include +#include +#include + +#include +#include +#include + +#include "gtest/gtest.h" +#include "sandboxed_api/sandbox2/util/fileops.h" + +namespace guetzli::sandbox::tests { + +namespace { + +constexpr absl::string_view kInPngFilename = "bees.png"; +constexpr absl::string_view kInJpegFilename = "nature.jpg"; +constexpr absl::string_view kOutJpegFilename = "out_jpeg.jpg"; +constexpr absl::string_view kOutPngFilename = "out_png.png"; +constexpr absl::string_view kPngReferenceFilename = "bees_reference.jpg"; +constexpr absl::string_view kJpegReferenceFIlename = "nature_reference.jpg"; + +constexpr int kPngExpectedSize = 38'625; +constexpr int kJpegExpectedSize = 10'816; + +constexpr int kDefaultQualityTarget = 95; +constexpr int kDefaultMemlimitMb = 6000; + +constexpr absl::string_view kRelativePathToTestdata = + "/guetzli_sandboxed/testdata/"; + +std::string GetPathToFile(absl::string_view filename) { + return absl::StrCat(getenv("TEST_SRCDIR"), kRelativePathToTestdata, filename); +} + +std::string ReadFromFile(const std::string& filename) { + std::ifstream stream(filename, std::ios::binary); + + if (!stream.is_open()) { + return ""; + } + + std::stringstream result; + result << stream.rdbuf(); + return result.str(); +} + +// Helper class to delete file after opening +class FileRemover { + public: + explicit FileRemover(const char* path) + : path_(path), fd_(open(path, O_RDONLY)) {} + + ~FileRemover() { + close(fd_); + remove(path_); + } + + int get() const { return fd_; } + + private: + const char* path_; + int fd_; +}; + +} // namespace + +TEST(GuetzliTransactionTest, TestTransactionJpg) { + std::string in_path = GetPathToFile(kInJpegFilename); + std::string out_path = GetPathToFile(kOutJpegFilename); + + TransactionParams params = {in_path.c_str(), out_path.c_str(), 0, + kDefaultQualityTarget, kDefaultMemlimitMb}; + { + GuetzliTransaction transaction(std::move(params)); + absl::Status result = transaction.Run(); + + ASSERT_TRUE(result.ok()) << result.ToString(); + } + std::string reference_data = + ReadFromFile(GetPathToFile(kJpegReferenceFIlename)); + FileRemover file_remover(out_path.c_str()); + ASSERT_TRUE(file_remover.get() != -1) << "Error opening output file"; + off_t output_size = lseek(file_remover.get(), 0, SEEK_END); + ASSERT_EQ(reference_data.size(), output_size) + << "Different sizes of reference and returned data"; + ASSERT_EQ(lseek(file_remover.get(), 0, SEEK_SET), 0) + << "Error repositioning out file"; + + std::string output; + output.resize(output_size); + ssize_t status = read(file_remover.get(), output.data(), output_size); + ASSERT_EQ(status, output_size) << "Error reading data from temp output file"; + + ASSERT_EQ(output, reference_data) << "Returned data doesn't match reference"; +} + +TEST(GuetzliTransactionTest, TestTransactionPng) { + std::string in_path = GetPathToFile(kInPngFilename); + std::string out_path = GetPathToFile(kOutPngFilename); + + TransactionParams params = {in_path.c_str(), out_path.c_str(), 0, + kDefaultQualityTarget, kDefaultMemlimitMb}; + { + GuetzliTransaction transaction(std::move(params)); + absl::Status result = transaction.Run(); + + ASSERT_TRUE(result.ok()) << result.ToString(); + } + std::string reference_data = + ReadFromFile(GetPathToFile(kPngReferenceFilename)); + FileRemover file_remover(out_path.c_str()); + ASSERT_TRUE(file_remover.get() != -1) << "Error opening output file"; + off_t output_size = lseek(file_remover.get(), 0, SEEK_END); + ASSERT_EQ(reference_data.size(), output_size) + << "Different sizes of reference and returned data"; + ASSERT_EQ(lseek(file_remover.get(), 0, SEEK_SET), 0) + << "Error repositioning out file"; + + std::string output; + output.resize(output_size); + ssize_t status = read(file_remover.get(), output.data(), output_size); + ASSERT_EQ(status, output_size) << "Error reading data from temp output file"; + + ASSERT_EQ(output, reference_data) << "Returned data doesn't match refernce"; +} + +} // namespace guetzli::sandbox::tests diff --git a/oss-internship-2020/guetzli/testdata/bees.png b/oss-internship-2020/guetzli/testdata/bees.png new file mode 100644 index 0000000..11640c7 Binary files /dev/null and b/oss-internship-2020/guetzli/testdata/bees.png differ diff --git a/oss-internship-2020/guetzli/testdata/bees_reference.jpg b/oss-internship-2020/guetzli/testdata/bees_reference.jpg new file mode 100644 index 0000000..2470067 Binary files /dev/null and b/oss-internship-2020/guetzli/testdata/bees_reference.jpg differ diff --git a/oss-internship-2020/guetzli/testdata/nature.jpg b/oss-internship-2020/guetzli/testdata/nature.jpg new file mode 100644 index 0000000..5c87177 Binary files /dev/null and b/oss-internship-2020/guetzli/testdata/nature.jpg differ diff --git a/oss-internship-2020/guetzli/testdata/nature_reference.jpg b/oss-internship-2020/guetzli/testdata/nature_reference.jpg new file mode 100644 index 0000000..b4f1c42 Binary files /dev/null and b/oss-internship-2020/guetzli/testdata/nature_reference.jpg differ diff --git a/oss-internship-2020/openjpeg/.gitignore b/oss-internship-2020/openjpeg/.gitignore new file mode 100644 index 0000000..567609b --- /dev/null +++ b/oss-internship-2020/openjpeg/.gitignore @@ -0,0 +1 @@ +build/ diff --git a/oss-internship-2020/openjpeg/CMakeLists.txt b/oss-internship-2020/openjpeg/CMakeLists.txt new file mode 100644 index 0000000..1a2748d --- /dev/null +++ b/oss-internship-2020/openjpeg/CMakeLists.txt @@ -0,0 +1,56 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http:#www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +cmake_minimum_required(VERSION 3.10) + +project(openjpeg-sapi C CXX) + +set(CMAKE_CXX_STANDARD 17) +set(CMAKE_CXX_STANDARD_REQUIRED True) + +# To override lib option -- else SAPI won't work +set(BUILD_SHARED_LIBS OFF CACHE BOOL "Build OpenJPEG shared library and link executables against it." FORCE) +add_subdirectory(openjpeg) + +set(SAPI_ROOT "../.." CACHE PATH "Path to the Sandboxed API source tree") +set(SAPI_ENABLE_EXAMPLES OFF CACHE BOOL "") +set(SAPI_ENABLE_TESTS OFF CACHE BOOL "") +set(EXECUTABLE_OUTPUT_PATH "" CACHE PATH "" FORCE) +add_subdirectory("${SAPI_ROOT}" + "${CMAKE_BINARY_DIR}/sandboxed-api-build" + EXCLUDE_FROM_ALL) + +add_sapi_library(openjp2_sapi + FUNCTIONS opj_stream_destroy + opj_stream_create_default_file_stream + opj_create_decompress + opj_image_destroy + opj_setup_decoder + opj_destroy_codec + opj_read_header + opj_decode + opj_set_default_decoder_parameters + opj_end_decompress + + INPUTS ${CMAKE_CURRENT_SOURCE_DIR}/openjpeg/src/lib/openjp2/openjpeg.h + LIBRARY openjp2 + LIBRARY_NAME Openjp2 + NAMESPACE "" +) + +target_include_directories(openjp2_sapi INTERFACE + "${PROJECT_BINARY_DIR}" +) + +add_subdirectory(examples) diff --git a/oss-internship-2020/openjpeg/README.md b/oss-internship-2020/openjpeg/README.md new file mode 100644 index 0000000..9a4a8d8 --- /dev/null +++ b/oss-internship-2020/openjpeg/README.md @@ -0,0 +1,29 @@ +# OpenJPEG Sandboxed API + +This library provides sandboxed version of the [OpenJPEG](https://github.com/uclouvain/openjpeg) library. + +## Examples + +The examples are sandboxed and simplified version of the main tools provided by the OpenJPEG library, namely (for now) `opj_decompress` from [here](https://github.com/uclouvain/openjpeg/blob/master/src/bin/jp2/opj_decompress.c). + +In `decompress_example.cc` the library's sandboxed API is used to convert the _.jp2_ to _.pnm_ image format. + +## Build + +To build this example, after cloning the whole Sandbox API project, you also need to run + +``` +git submodule update --init --recursive +``` +anywhere in the project tree in order to clone the `openjpeg` submodule. +Then in the `sandboxed-api/oss-internship-2020/openjpeg` run +``` +mkdir build && cd build +cmake -G Ninja +ninja +``` +To run `decompress_sandboxed`: +``` +cd examples +./decompress_sandboxed absolute/path/to/the/file.jp2 absolute/path/to/the/file.pnm +``` diff --git a/oss-internship-2020/openjpeg/examples/CMakeLists.txt b/oss-internship-2020/openjpeg/examples/CMakeLists.txt new file mode 100644 index 0000000..82b3cf4 --- /dev/null +++ b/oss-internship-2020/openjpeg/examples/CMakeLists.txt @@ -0,0 +1,48 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http:#www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# we need to use modified versions of some of the library tools + +file(MAKE_DIRECTORY "${PROJECT_BINARY_DIR}/gen_files") +file(COPY "${PROJECT_SOURCE_DIR}/openjpeg/src/bin/jp2/convert.c" DESTINATION "${PROJECT_BINARY_DIR}/gen_files/") +file(COPY "${PROJECT_SOURCE_DIR}/openjpeg/src/bin/jp2/convert.h" DESTINATION "${PROJECT_BINARY_DIR}/gen_files/") +file(COPY "${PROJECT_SOURCE_DIR}/examples/convert.patch" DESTINATION "${PROJECT_BINARY_DIR}/gen_files/") +file(COPY "${PROJECT_SOURCE_DIR}/examples/convert_h.patch" DESTINATION "${PROJECT_BINARY_DIR}/gen_files/") + +add_custom_command( + OUTPUT ${PROJECT_BINARY_DIR}/gen_files/convert.cc ${PROJECT_BINARY_DIR}/gen_files/convert.h + COMMAND cd ${PROJECT_BINARY_DIR}/gen_files && patch < ${PROJECT_SOURCE_DIR}/examples/convert.patch > /dev/null + COMMAND cd ${PROJECT_BINARY_DIR}/gen_files && patch < ${PROJECT_SOURCE_DIR}/examples/convert_h.patch > /dev/null + COMMAND mv ${PROJECT_BINARY_DIR}/gen_files/convert.c ${PROJECT_BINARY_DIR}/gen_files/convert.cc +) + +add_library(convert_helper STATIC + ${PROJECT_BINARY_DIR}/gen_files/convert.cc + ${PROJECT_BINARY_DIR}/gen_files/convert.h +) + +add_executable(decompress_sandboxed + decompress_example.cc +) + +target_link_libraries(decompress_sandboxed PRIVATE + convert_helper + openjp2_sapi + sapi::sapi +) + +target_link_libraries(convert_helper PRIVATE + openjp2_sapi + sapi::sapi +) diff --git a/oss-internship-2020/openjpeg/examples/convert.patch b/oss-internship-2020/openjpeg/examples/convert.patch new file mode 100644 index 0000000..15bc1a3 --- /dev/null +++ b/oss-internship-2020/openjpeg/examples/convert.patch @@ -0,0 +1,2871 @@ +--- convert.c 2020-08-27 15:46:32.911584344 +0000 ++++ convert_helper.cc 2020-08-27 15:10:52.695231032 +0000 +@@ -35,2605 +35,311 @@ + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ +-#include "opj_apps_config.h" + +-#include +-#include +-#include +-#include +-#include +- +-#include "openjpeg.h" +-#include "convert.h" +- +-/* +- * Get logarithm of an integer and round downwards. +- * +- * log2(a) +- */ +-static int int_floorlog2(int a) +-{ +- int l; +- for (l = 0; a > 1; l++) { +- a >>= 1; +- } +- return l; +-} +- +-/* Component precision scaling */ +-void clip_component(opj_image_comp_t* component, OPJ_UINT32 precision) +-{ +- OPJ_SIZE_T i; +- OPJ_SIZE_T len; +- OPJ_UINT32 umax = (OPJ_UINT32)((OPJ_INT32) - 1); +- +- len = (OPJ_SIZE_T)component->w * (OPJ_SIZE_T)component->h; +- if (precision < 32) { +- umax = (1U << precision) - 1U; +- } +- +- if (component->sgnd) { +- OPJ_INT32* l_data = component->data; +- OPJ_INT32 max = (OPJ_INT32)(umax / 2U); +- OPJ_INT32 min = -max - 1; +- for (i = 0; i < len; ++i) { +- if (l_data[i] > max) { +- l_data[i] = max; +- } else if (l_data[i] < min) { +- l_data[i] = min; +- } +- } +- } else { +- OPJ_UINT32* l_data = (OPJ_UINT32*)component->data; +- for (i = 0; i < len; ++i) { +- if (l_data[i] > umax) { +- l_data[i] = umax; +- } +- } +- } +- component->prec = precision; +-} ++// patched versions of a few library tools + +-/* Component precision scaling */ +-static void scale_component_up(opj_image_comp_t* component, +- OPJ_UINT32 precision) +-{ +- OPJ_SIZE_T i, len; +- +- len = (OPJ_SIZE_T)component->w * (OPJ_SIZE_T)component->h; +- if (component->sgnd) { +- OPJ_INT64 newMax = (OPJ_INT64)(1U << (precision - 1)); +- OPJ_INT64 oldMax = (OPJ_INT64)(1U << (component->prec - 1)); +- OPJ_INT32* l_data = component->data; +- for (i = 0; i < len; ++i) { +- l_data[i] = (OPJ_INT32)(((OPJ_INT64)l_data[i] * newMax) / oldMax); +- } +- } else { +- OPJ_UINT64 newMax = (OPJ_UINT64)((1U << precision) - 1U); +- OPJ_UINT64 oldMax = (OPJ_UINT64)((1U << component->prec) - 1U); +- OPJ_UINT32* l_data = (OPJ_UINT32*)component->data; +- for (i = 0; i < len; ++i) { +- l_data[i] = (OPJ_UINT32)(((OPJ_UINT64)l_data[i] * newMax) / oldMax); +- } +- } +- component->prec = precision; +- component->bpp = precision; +-} +-void scale_component(opj_image_comp_t* component, OPJ_UINT32 precision) +-{ +- int shift; +- OPJ_SIZE_T i, len; +- +- if (component->prec == precision) { +- return; +- } +- if (component->prec < precision) { +- scale_component_up(component, precision); +- return; +- } +- shift = (int)(component->prec - precision); +- len = (OPJ_SIZE_T)component->w * (OPJ_SIZE_T)component->h; +- if (component->sgnd) { +- OPJ_INT32* l_data = component->data; +- for (i = 0; i < len; ++i) { +- l_data[i] >>= shift; +- } +- } else { +- OPJ_UINT32* l_data = (OPJ_UINT32*)component->data; +- for (i = 0; i < len; ++i) { +- l_data[i] >>= shift; +- } +- } +- component->bpp = precision; +- component->prec = precision; +-} +- +- +-/* planar / interleaved conversions */ +-/* used by PNG/TIFF */ +-static void convert_32s_C1P1(const OPJ_INT32* pSrc, OPJ_INT32* const* pDst, +- OPJ_SIZE_T length) +-{ +- memcpy(pDst[0], pSrc, length * sizeof(OPJ_INT32)); +-} +-static void convert_32s_C2P2(const OPJ_INT32* pSrc, OPJ_INT32* const* pDst, +- OPJ_SIZE_T length) +-{ +- OPJ_SIZE_T i; +- OPJ_INT32* pDst0 = pDst[0]; +- OPJ_INT32* pDst1 = pDst[1]; +- +- for (i = 0; i < length; i++) { +- pDst0[i] = pSrc[2 * i + 0]; +- pDst1[i] = pSrc[2 * i + 1]; +- } +-} +-static void convert_32s_C3P3(const OPJ_INT32* pSrc, OPJ_INT32* const* pDst, +- OPJ_SIZE_T length) +-{ +- OPJ_SIZE_T i; +- OPJ_INT32* pDst0 = pDst[0]; +- OPJ_INT32* pDst1 = pDst[1]; +- OPJ_INT32* pDst2 = pDst[2]; +- +- for (i = 0; i < length; i++) { +- pDst0[i] = pSrc[3 * i + 0]; +- pDst1[i] = pSrc[3 * i + 1]; +- pDst2[i] = pSrc[3 * i + 2]; +- } +-} +-static void convert_32s_C4P4(const OPJ_INT32* pSrc, OPJ_INT32* const* pDst, +- OPJ_SIZE_T length) +-{ +- OPJ_SIZE_T i; +- OPJ_INT32* pDst0 = pDst[0]; +- OPJ_INT32* pDst1 = pDst[1]; +- OPJ_INT32* pDst2 = pDst[2]; +- OPJ_INT32* pDst3 = pDst[3]; +- +- for (i = 0; i < length; i++) { +- pDst0[i] = pSrc[4 * i + 0]; +- pDst1[i] = pSrc[4 * i + 1]; +- pDst2[i] = pSrc[4 * i + 2]; +- pDst3[i] = pSrc[4 * i + 3]; +- } +-} +-const convert_32s_CXPX convert_32s_CXPX_LUT[5] = { +- NULL, +- convert_32s_C1P1, +- convert_32s_C2P2, +- convert_32s_C3P3, +- convert_32s_C4P4 +-}; +- +-static void convert_32s_P1C1(OPJ_INT32 const* const* pSrc, OPJ_INT32* pDst, +- OPJ_SIZE_T length, OPJ_INT32 adjust) +-{ +- OPJ_SIZE_T i; +- const OPJ_INT32* pSrc0 = pSrc[0]; ++#include "convert.h" + +- for (i = 0; i < length; i++) { +- pDst[i] = pSrc0[i] + adjust; +- } +-} +-static void convert_32s_P2C2(OPJ_INT32 const* const* pSrc, OPJ_INT32* pDst, +- OPJ_SIZE_T length, OPJ_INT32 adjust) +-{ +- OPJ_SIZE_T i; +- const OPJ_INT32* pSrc0 = pSrc[0]; +- const OPJ_INT32* pSrc1 = pSrc[1]; +- +- for (i = 0; i < length; i++) { +- pDst[2 * i + 0] = pSrc0[i] + adjust; +- pDst[2 * i + 1] = pSrc1[i] + adjust; +- } +-} +-static void convert_32s_P3C3(OPJ_INT32 const* const* pSrc, OPJ_INT32* pDst, +- OPJ_SIZE_T length, OPJ_INT32 adjust) +-{ +- OPJ_SIZE_T i; +- const OPJ_INT32* pSrc0 = pSrc[0]; +- const OPJ_INT32* pSrc1 = pSrc[1]; +- const OPJ_INT32* pSrc2 = pSrc[2]; +- +- for (i = 0; i < length; i++) { +- pDst[3 * i + 0] = pSrc0[i] + adjust; +- pDst[3 * i + 1] = pSrc1[i] + adjust; +- pDst[3 * i + 2] = pSrc2[i] + adjust; +- } +-} +-static void convert_32s_P4C4(OPJ_INT32 const* const* pSrc, OPJ_INT32* pDst, +- OPJ_SIZE_T length, OPJ_INT32 adjust) +-{ +- OPJ_SIZE_T i; +- const OPJ_INT32* pSrc0 = pSrc[0]; +- const OPJ_INT32* pSrc1 = pSrc[1]; +- const OPJ_INT32* pSrc2 = pSrc[2]; +- const OPJ_INT32* pSrc3 = pSrc[3]; +- +- for (i = 0; i < length; i++) { +- pDst[4 * i + 0] = pSrc0[i] + adjust; +- pDst[4 * i + 1] = pSrc1[i] + adjust; +- pDst[4 * i + 2] = pSrc2[i] + adjust; +- pDst[4 * i + 3] = pSrc3[i] + adjust; +- } +-} +-const convert_32s_PXCX convert_32s_PXCX_LUT[5] = { +- NULL, +- convert_32s_P1C1, +- convert_32s_P2C2, +- convert_32s_P3C3, +- convert_32s_P4C4 +-}; +- +-/* bit depth conversions */ +-/* used by PNG/TIFF up to 8bpp */ +-static void convert_1u32s_C1R(const OPJ_BYTE* pSrc, OPJ_INT32* pDst, +- OPJ_SIZE_T length) +-{ +- OPJ_SIZE_T i; +- for (i = 0; i < (length & ~(OPJ_SIZE_T)7U); i += 8U) { +- OPJ_UINT32 val = *pSrc++; +- pDst[i + 0] = (OPJ_INT32)(val >> 7); +- pDst[i + 1] = (OPJ_INT32)((val >> 6) & 0x1U); +- pDst[i + 2] = (OPJ_INT32)((val >> 5) & 0x1U); +- pDst[i + 3] = (OPJ_INT32)((val >> 4) & 0x1U); +- pDst[i + 4] = (OPJ_INT32)((val >> 3) & 0x1U); +- pDst[i + 5] = (OPJ_INT32)((val >> 2) & 0x1U); +- pDst[i + 6] = (OPJ_INT32)((val >> 1) & 0x1U); +- pDst[i + 7] = (OPJ_INT32)(val & 0x1U); +- } +- if (length & 7U) { +- OPJ_UINT32 val = *pSrc++; +- length = length & 7U; +- pDst[i + 0] = (OPJ_INT32)(val >> 7); +- +- if (length > 1U) { +- pDst[i + 1] = (OPJ_INT32)((val >> 6) & 0x1U); +- if (length > 2U) { +- pDst[i + 2] = (OPJ_INT32)((val >> 5) & 0x1U); +- if (length > 3U) { +- pDst[i + 3] = (OPJ_INT32)((val >> 4) & 0x1U); +- if (length > 4U) { +- pDst[i + 4] = (OPJ_INT32)((val >> 3) & 0x1U); +- if (length > 5U) { +- pDst[i + 5] = (OPJ_INT32)((val >> 2) & 0x1U); +- if (length > 6U) { +- pDst[i + 6] = (OPJ_INT32)((val >> 1) & 0x1U); +- } +- } +- } +- } +- } +- } +- } +-} +-static void convert_2u32s_C1R(const OPJ_BYTE* pSrc, OPJ_INT32* pDst, +- OPJ_SIZE_T length) +-{ +- OPJ_SIZE_T i; +- for (i = 0; i < (length & ~(OPJ_SIZE_T)3U); i += 4U) { +- OPJ_UINT32 val = *pSrc++; +- pDst[i + 0] = (OPJ_INT32)(val >> 6); +- pDst[i + 1] = (OPJ_INT32)((val >> 4) & 0x3U); +- pDst[i + 2] = (OPJ_INT32)((val >> 2) & 0x3U); +- pDst[i + 3] = (OPJ_INT32)(val & 0x3U); +- } +- if (length & 3U) { +- OPJ_UINT32 val = *pSrc++; +- length = length & 3U; +- pDst[i + 0] = (OPJ_INT32)(val >> 6); +- +- if (length > 1U) { +- pDst[i + 1] = (OPJ_INT32)((val >> 4) & 0x3U); +- if (length > 2U) { +- pDst[i + 2] = (OPJ_INT32)((val >> 2) & 0x3U); ++#define OPJ_TRUE 1 ++#define OPJ_FALSE 0 + +- } +- } +- } +-} +-static void convert_4u32s_C1R(const OPJ_BYTE* pSrc, OPJ_INT32* pDst, +- OPJ_SIZE_T length) +-{ +- OPJ_SIZE_T i; +- for (i = 0; i < (length & ~(OPJ_SIZE_T)1U); i += 2U) { +- OPJ_UINT32 val = *pSrc++; +- pDst[i + 0] = (OPJ_INT32)(val >> 4); +- pDst[i + 1] = (OPJ_INT32)(val & 0xFU); +- } +- if (length & 1U) { +- OPJ_UINT8 val = *pSrc++; +- pDst[i + 0] = (OPJ_INT32)(val >> 4); +- } +-} +-static void convert_6u32s_C1R(const OPJ_BYTE* pSrc, OPJ_INT32* pDst, +- OPJ_SIZE_T length) +-{ +- OPJ_SIZE_T i; +- for (i = 0; i < (length & ~(OPJ_SIZE_T)3U); i += 4U) { +- OPJ_UINT32 val0 = *pSrc++; +- OPJ_UINT32 val1 = *pSrc++; +- OPJ_UINT32 val2 = *pSrc++; +- pDst[i + 0] = (OPJ_INT32)(val0 >> 2); +- pDst[i + 1] = (OPJ_INT32)(((val0 & 0x3U) << 4) | (val1 >> 4)); +- pDst[i + 2] = (OPJ_INT32)(((val1 & 0xFU) << 2) | (val2 >> 6)); +- pDst[i + 3] = (OPJ_INT32)(val2 & 0x3FU); ++const char *opj_version(void) { return "2.3.1"; } + +- } +- if (length & 3U) { +- OPJ_UINT32 val0 = *pSrc++; +- length = length & 3U; +- pDst[i + 0] = (OPJ_INT32)(val0 >> 2); +- +- if (length > 1U) { +- OPJ_UINT32 val1 = *pSrc++; +- pDst[i + 1] = (OPJ_INT32)(((val0 & 0x3U) << 4) | (val1 >> 4)); +- if (length > 2U) { +- OPJ_UINT32 val2 = *pSrc++; +- pDst[i + 2] = (OPJ_INT32)(((val1 & 0xFU) << 2) | (val2 >> 6)); +- } +- } +- } +-} +-static void convert_8u32s_C1R(const OPJ_BYTE* pSrc, OPJ_INT32* pDst, +- OPJ_SIZE_T length) +-{ +- OPJ_SIZE_T i; +- for (i = 0; i < length; i++) { +- pDst[i] = pSrc[i]; +- } +-} +-const convert_XXx32s_C1R convert_XXu32s_C1R_LUT[9] = { +- NULL, +- convert_1u32s_C1R, +- convert_2u32s_C1R, +- NULL, +- convert_4u32s_C1R, +- NULL, +- convert_6u32s_C1R, +- NULL, +- convert_8u32s_C1R +-}; +- +- +-static void convert_32s1u_C1R(const OPJ_INT32* pSrc, OPJ_BYTE* pDst, +- OPJ_SIZE_T length) +-{ +- OPJ_SIZE_T i; +- for (i = 0; i < (length & ~(OPJ_SIZE_T)7U); i += 8U) { +- OPJ_UINT32 src0 = (OPJ_UINT32)pSrc[i + 0]; +- OPJ_UINT32 src1 = (OPJ_UINT32)pSrc[i + 1]; +- OPJ_UINT32 src2 = (OPJ_UINT32)pSrc[i + 2]; +- OPJ_UINT32 src3 = (OPJ_UINT32)pSrc[i + 3]; +- OPJ_UINT32 src4 = (OPJ_UINT32)pSrc[i + 4]; +- OPJ_UINT32 src5 = (OPJ_UINT32)pSrc[i + 5]; +- OPJ_UINT32 src6 = (OPJ_UINT32)pSrc[i + 6]; +- OPJ_UINT32 src7 = (OPJ_UINT32)pSrc[i + 7]; +- +- *pDst++ = (OPJ_BYTE)((src0 << 7) | (src1 << 6) | (src2 << 5) | (src3 << 4) | +- (src4 << 3) | (src5 << 2) | (src6 << 1) | src7); +- } +- +- if (length & 7U) { +- OPJ_UINT32 src0 = (OPJ_UINT32)pSrc[i + 0]; +- OPJ_UINT32 src1 = 0U; +- OPJ_UINT32 src2 = 0U; +- OPJ_UINT32 src3 = 0U; +- OPJ_UINT32 src4 = 0U; +- OPJ_UINT32 src5 = 0U; +- OPJ_UINT32 src6 = 0U; +- length = length & 7U; +- +- if (length > 1U) { +- src1 = (OPJ_UINT32)pSrc[i + 1]; +- if (length > 2U) { +- src2 = (OPJ_UINT32)pSrc[i + 2]; +- if (length > 3U) { +- src3 = (OPJ_UINT32)pSrc[i + 3]; +- if (length > 4U) { +- src4 = (OPJ_UINT32)pSrc[i + 4]; +- if (length > 5U) { +- src5 = (OPJ_UINT32)pSrc[i + 5]; +- if (length > 6U) { +- src6 = (OPJ_UINT32)pSrc[i + 6]; +- } +- } +- } +- } +- } +- } +- *pDst++ = (OPJ_BYTE)((src0 << 7) | (src1 << 6) | (src2 << 5) | (src3 << 4) | +- (src4 << 3) | (src5 << 2) | (src6 << 1)); +- } +-} +- +-static void convert_32s2u_C1R(const OPJ_INT32* pSrc, OPJ_BYTE* pDst, +- OPJ_SIZE_T length) +-{ +- OPJ_SIZE_T i; +- for (i = 0; i < (length & ~(OPJ_SIZE_T)3U); i += 4U) { +- OPJ_UINT32 src0 = (OPJ_UINT32)pSrc[i + 0]; +- OPJ_UINT32 src1 = (OPJ_UINT32)pSrc[i + 1]; +- OPJ_UINT32 src2 = (OPJ_UINT32)pSrc[i + 2]; +- OPJ_UINT32 src3 = (OPJ_UINT32)pSrc[i + 3]; +- +- *pDst++ = (OPJ_BYTE)((src0 << 6) | (src1 << 4) | (src2 << 2) | src3); +- } +- +- if (length & 3U) { +- OPJ_UINT32 src0 = (OPJ_UINT32)pSrc[i + 0]; +- OPJ_UINT32 src1 = 0U; +- OPJ_UINT32 src2 = 0U; +- length = length & 3U; +- +- if (length > 1U) { +- src1 = (OPJ_UINT32)pSrc[i + 1]; +- if (length > 2U) { +- src2 = (OPJ_UINT32)pSrc[i + 2]; +- } +- } +- *pDst++ = (OPJ_BYTE)((src0 << 6) | (src1 << 4) | (src2 << 2)); +- } +-} +- +-static void convert_32s4u_C1R(const OPJ_INT32* pSrc, OPJ_BYTE* pDst, +- OPJ_SIZE_T length) +-{ +- OPJ_SIZE_T i; +- for (i = 0; i < (length & ~(OPJ_SIZE_T)1U); i += 2U) { +- OPJ_UINT32 src0 = (OPJ_UINT32)pSrc[i + 0]; +- OPJ_UINT32 src1 = (OPJ_UINT32)pSrc[i + 1]; +- +- *pDst++ = (OPJ_BYTE)((src0 << 4) | src1); +- } +- +- if (length & 1U) { +- OPJ_UINT32 src0 = (OPJ_UINT32)pSrc[i + 0]; +- *pDst++ = (OPJ_BYTE)((src0 << 4)); +- } +-} +- +-static void convert_32s6u_C1R(const OPJ_INT32* pSrc, OPJ_BYTE* pDst, +- OPJ_SIZE_T length) +-{ +- OPJ_SIZE_T i; +- for (i = 0; i < (length & ~(OPJ_SIZE_T)3U); i += 4U) { +- OPJ_UINT32 src0 = (OPJ_UINT32)pSrc[i + 0]; +- OPJ_UINT32 src1 = (OPJ_UINT32)pSrc[i + 1]; +- OPJ_UINT32 src2 = (OPJ_UINT32)pSrc[i + 2]; +- OPJ_UINT32 src3 = (OPJ_UINT32)pSrc[i + 3]; +- +- *pDst++ = (OPJ_BYTE)((src0 << 2) | (src1 >> 4)); +- *pDst++ = (OPJ_BYTE)(((src1 & 0xFU) << 4) | (src2 >> 2)); +- *pDst++ = (OPJ_BYTE)(((src2 & 0x3U) << 6) | src3); +- } +- +- if (length & 3U) { +- OPJ_UINT32 src0 = (OPJ_UINT32)pSrc[i + 0]; +- OPJ_UINT32 src1 = 0U; +- OPJ_UINT32 src2 = 0U; +- length = length & 3U; +- +- if (length > 1U) { +- src1 = (OPJ_UINT32)pSrc[i + 1]; +- if (length > 2U) { +- src2 = (OPJ_UINT32)pSrc[i + 2]; +- } +- } +- *pDst++ = (OPJ_BYTE)((src0 << 2) | (src1 >> 4)); +- if (length > 1U) { +- *pDst++ = (OPJ_BYTE)(((src1 & 0xFU) << 4) | (src2 >> 2)); +- if (length > 2U) { +- *pDst++ = (OPJ_BYTE)(((src2 & 0x3U) << 6)); +- } +- } +- } +-} +-static void convert_32s8u_C1R(const OPJ_INT32* pSrc, OPJ_BYTE* pDst, +- OPJ_SIZE_T length) +-{ +- OPJ_SIZE_T i; +- for (i = 0; i < length; ++i) { +- pDst[i] = (OPJ_BYTE)pSrc[i]; +- } +-} +-const convert_32sXXx_C1R convert_32sXXu_C1R_LUT[9] = { +- NULL, +- convert_32s1u_C1R, +- convert_32s2u_C1R, +- NULL, +- convert_32s4u_C1R, +- NULL, +- convert_32s6u_C1R, +- NULL, +- convert_32s8u_C1R +-}; +- +-/* -->> -->> -->> -->> +- +- TGA IMAGE FORMAT +- +- <<-- <<-- <<-- <<-- */ +- +-#ifdef INFORMATION_ONLY +-/* TGA header definition. */ +-struct tga_header { +- unsigned char id_length; /* Image id field length */ +- unsigned char colour_map_type; /* Colour map type */ +- unsigned char image_type; /* Image type */ +- /* +- ** Colour map specification +- */ +- unsigned short colour_map_index; /* First entry index */ +- unsigned short colour_map_length; /* Colour map length */ +- unsigned char colour_map_entry_size; /* Colour map entry size */ +- /* +- ** Image specification +- */ +- unsigned short x_origin; /* x origin of image */ +- unsigned short y_origin; /* u origin of image */ +- unsigned short image_width; /* Image width */ +- unsigned short image_height; /* Image height */ +- unsigned char pixel_depth; /* Pixel depth */ +- unsigned char image_desc; /* Image descriptor */ +-}; +-#endif /* INFORMATION_ONLY */ +- +-/* Returns a ushort from a little-endian serialized value */ +-static unsigned short get_tga_ushort(const unsigned char *data) +-{ +- return (unsigned short)(data[0] | (data[1] << 8)); +-} +- +-#define TGA_HEADER_SIZE 18 +- +-static int tga_readheader(FILE *fp, unsigned int *bits_per_pixel, +- unsigned int *width, unsigned int *height, int *flip_image) +-{ +- int palette_size; +- unsigned char tga[TGA_HEADER_SIZE]; +- unsigned char id_len, /*cmap_type,*/ image_type; +- unsigned char pixel_depth, image_desc; +- unsigned short /*cmap_index,*/ cmap_len, cmap_entry_size; +- unsigned short /*x_origin, y_origin,*/ image_w, image_h; +- +- if (!bits_per_pixel || !width || !height || !flip_image) { +- return 0; +- } +- +- if (fread(tga, TGA_HEADER_SIZE, 1, fp) != 1) { +- fprintf(stderr, +- "\nError: fread return a number of element different from the expected.\n"); +- return 0 ; +- } +- id_len = tga[0]; +- /*cmap_type = tga[1];*/ +- image_type = tga[2]; +- /*cmap_index = get_tga_ushort(&tga[3]);*/ +- cmap_len = get_tga_ushort(&tga[5]); +- cmap_entry_size = tga[7]; +- +- +-#if 0 +- x_origin = get_tga_ushort(&tga[8]); +- y_origin = get_tga_ushort(&tga[10]); +-#endif +- image_w = get_tga_ushort(&tga[12]); +- image_h = get_tga_ushort(&tga[14]); +- pixel_depth = tga[16]; +- image_desc = tga[17]; +- +- *bits_per_pixel = (unsigned int)pixel_depth; +- *width = (unsigned int)image_w; +- *height = (unsigned int)image_h; +- +- /* Ignore tga identifier, if present ... */ +- if (id_len) { +- unsigned char *id = (unsigned char *) malloc(id_len); +- if (id == 0) { +- fprintf(stderr, "tga_readheader: memory out\n"); +- return 0; +- } +- if (!fread(id, id_len, 1, fp)) { +- fprintf(stderr, +- "\nError: fread return a number of element different from the expected.\n"); +- free(id); +- return 0 ; +- } +- free(id); +- } +- +- /* Test for compressed formats ... not yet supported ... +- // Note :- 9 - RLE encoded palettized. +- // 10 - RLE encoded RGB. */ +- if (image_type > 8) { +- fprintf(stderr, "Sorry, compressed tga files are not currently supported.\n"); +- return 0 ; +- } +- +- *flip_image = !(image_desc & 32); +- +- /* Palettized formats are not yet supported, skip over the palette, if present ... */ +- palette_size = cmap_len * (cmap_entry_size / 8); +- +- if (palette_size > 0) { +- fprintf(stderr, "File contains a palette - not yet supported."); +- fseek(fp, palette_size, SEEK_CUR); +- } ++static int are_comps_similar(opj_image_t *image) { ++ unsigned int i; ++ for (i = 1; i < image->numcomps; i++) { ++ if (image->comps[0].dx != image->comps[i].dx || ++ image->comps[0].dy != image->comps[i].dy || ++ (i <= 2 && (image->comps[0].prec != image->comps[i].prec || ++ image->comps[0].sgnd != image->comps[i].sgnd))) { ++ return OPJ_FALSE; ++ } ++ } ++ return OPJ_TRUE; ++} ++ ++int imagetopnm(opj_image_t *image, const char *outfile, int force_split) { ++ int *red, *green, *blue, *alpha; ++ int wr, hr, max; ++ int i; ++ unsigned int compno, ncomp; ++ int adjustR, adjustG, adjustB, adjustA; ++ int fails, two, want_gray, has_alpha, triple; ++ int prec, v; ++ FILE *fdest = NULL; ++ const char *tmp = outfile; ++ char *destname; ++ ++ alpha = NULL; ++ ++ if ((prec = (int)image->comps[0].prec) > 16) { ++ fprintf(stderr, ++ "%s:%d:imagetopnm\n\tprecision %d is larger than 16" ++ "\n\t: refused.\n", ++ __FILE__, __LINE__, prec); + return 1; +-} +- +-#ifdef OPJ_BIG_ENDIAN +- +-static INLINE OPJ_UINT16 swap16(OPJ_UINT16 x) +-{ +- return (OPJ_UINT16)(((x & 0x00ffU) << 8) | ((x & 0xff00U) >> 8)); +-} +- +-#endif ++ } + +-static int tga_writeheader(FILE *fp, int bits_per_pixel, int width, int height, +- OPJ_BOOL flip_image) +-{ +- OPJ_UINT16 image_w, image_h, us0; +- unsigned char uc0, image_type; +- unsigned char pixel_depth, image_desc; +- +- if (!bits_per_pixel || !width || !height) { +- return 0; +- } +- +- pixel_depth = 0; +- +- if (bits_per_pixel < 256) { +- pixel_depth = (unsigned char)bits_per_pixel; +- } else { +- fprintf(stderr, "ERROR: Wrong bits per pixel inside tga_header"); +- return 0; +- } +- uc0 = 0; +- +- if (fwrite(&uc0, 1, 1, fp) != 1) { +- goto fails; /* id_length */ +- } +- if (fwrite(&uc0, 1, 1, fp) != 1) { +- goto fails; /* colour_map_type */ +- } +- +- image_type = 2; /* Uncompressed. */ +- if (fwrite(&image_type, 1, 1, fp) != 1) { +- goto fails; +- } +- +- us0 = 0; +- if (fwrite(&us0, 2, 1, fp) != 1) { +- goto fails; /* colour_map_index */ +- } +- if (fwrite(&us0, 2, 1, fp) != 1) { +- goto fails; /* colour_map_length */ +- } +- if (fwrite(&uc0, 1, 1, fp) != 1) { +- goto fails; /* colour_map_entry_size */ +- } +- +- if (fwrite(&us0, 2, 1, fp) != 1) { +- goto fails; /* x_origin */ +- } +- if (fwrite(&us0, 2, 1, fp) != 1) { +- goto fails; /* y_origin */ +- } +- +- image_w = (unsigned short)width; +- image_h = (unsigned short) height; +- +-#ifndef OPJ_BIG_ENDIAN +- if (fwrite(&image_w, 2, 1, fp) != 1) { +- goto fails; +- } +- if (fwrite(&image_h, 2, 1, fp) != 1) { +- goto fails; +- } +-#else +- image_w = swap16(image_w); +- image_h = swap16(image_h); +- if (fwrite(&image_w, 2, 1, fp) != 1) { +- goto fails; +- } +- if (fwrite(&image_h, 2, 1, fp) != 1) { +- goto fails; +- } +-#endif +- +- if (fwrite(&pixel_depth, 1, 1, fp) != 1) { +- goto fails; +- } +- +- image_desc = 8; /* 8 bits per component. */ +- +- if (flip_image) { +- image_desc |= 32; +- } +- if (fwrite(&image_desc, 1, 1, fp) != 1) { +- goto fails; +- } +- +- return 1; +- +-fails: +- fputs("\nwrite_tgaheader: write ERROR\n", stderr); +- return 0; +-} +- +-opj_image_t* tgatoimage(const char *filename, opj_cparameters_t *parameters) +-{ +- FILE *f; +- opj_image_t *image; +- unsigned int image_width, image_height, pixel_bit_depth; +- unsigned int x, y; +- int flip_image = 0; +- opj_image_cmptparm_t cmptparm[4]; /* maximum 4 components */ +- int numcomps; +- OPJ_COLOR_SPACE color_space; +- OPJ_BOOL mono ; +- OPJ_BOOL save_alpha; +- int subsampling_dx, subsampling_dy; +- int i; +- +- f = fopen(filename, "rb"); +- if (!f) { +- fprintf(stderr, "Failed to open %s for reading !!\n", filename); +- return 0; +- } +- +- if (!tga_readheader(f, &pixel_bit_depth, &image_width, &image_height, +- &flip_image)) { +- fclose(f); +- return NULL; +- } +- +- /* We currently only support 24 & 32 bit tga's ... */ +- if (!((pixel_bit_depth == 24) || (pixel_bit_depth == 32))) { +- fclose(f); +- return NULL; +- } +- +- /* initialize image components */ +- memset(&cmptparm[0], 0, 4 * sizeof(opj_image_cmptparm_t)); +- +- mono = (pixel_bit_depth == 8) || +- (pixel_bit_depth == 16); /* Mono with & without alpha. */ +- save_alpha = (pixel_bit_depth == 16) || +- (pixel_bit_depth == 32); /* Mono with alpha, or RGB with alpha */ +- +- if (mono) { +- color_space = OPJ_CLRSPC_GRAY; +- numcomps = save_alpha ? 2 : 1; +- } else { +- numcomps = save_alpha ? 4 : 3; +- color_space = OPJ_CLRSPC_SRGB; +- } +- +- /* If the declared file size is > 10 MB, check that the file is big */ +- /* enough to avoid excessive memory allocations */ +- if (image_height != 0 && +- image_width > 10000000U / image_height / (OPJ_UINT32)numcomps) { +- char ch; +- OPJ_UINT64 expected_file_size = +- (OPJ_UINT64)image_width * image_height * (OPJ_UINT32)numcomps; +- long curpos = ftell(f); +- if (expected_file_size > (OPJ_UINT64)INT_MAX) { +- expected_file_size = (OPJ_UINT64)INT_MAX; +- } +- fseek(f, (long)expected_file_size - 1, SEEK_SET); +- if (fread(&ch, 1, 1, f) != 1) { +- fclose(f); +- return NULL; +- } +- fseek(f, curpos, SEEK_SET); +- } +- +- subsampling_dx = parameters->subsampling_dx; +- subsampling_dy = parameters->subsampling_dy; +- +- for (i = 0; i < numcomps; i++) { +- cmptparm[i].prec = 8; +- cmptparm[i].bpp = 8; +- cmptparm[i].sgnd = 0; +- cmptparm[i].dx = (OPJ_UINT32)subsampling_dx; +- cmptparm[i].dy = (OPJ_UINT32)subsampling_dy; +- cmptparm[i].w = image_width; +- cmptparm[i].h = image_height; +- } +- +- /* create the image */ +- image = opj_image_create((OPJ_UINT32)numcomps, &cmptparm[0], color_space); +- +- if (!image) { +- fclose(f); +- return NULL; +- } +- +- +- /* set image offset and reference grid */ +- image->x0 = (OPJ_UINT32)parameters->image_offset_x0; +- image->y0 = (OPJ_UINT32)parameters->image_offset_y0; +- image->x1 = !image->x0 ? (OPJ_UINT32)(image_width - 1) * +- (OPJ_UINT32)subsampling_dx + 1 : image->x0 + (OPJ_UINT32)(image_width - 1) * +- (OPJ_UINT32)subsampling_dx + 1; +- image->y1 = !image->y0 ? (OPJ_UINT32)(image_height - 1) * +- (OPJ_UINT32)subsampling_dy + 1 : image->y0 + (OPJ_UINT32)(image_height - 1) * +- (OPJ_UINT32)subsampling_dy + 1; +- +- /* set image data */ +- for (y = 0; y < image_height; y++) { +- int index; +- +- if (flip_image) { +- index = (int)((image_height - y - 1) * image_width); +- } else { +- index = (int)(y * image_width); +- } +- +- if (numcomps == 3) { +- for (x = 0; x < image_width; x++) { +- unsigned char r, g, b; +- +- if (!fread(&b, 1, 1, f)) { +- fprintf(stderr, +- "\nError: fread return a number of element different from the expected.\n"); +- opj_image_destroy(image); +- fclose(f); +- return NULL; +- } +- if (!fread(&g, 1, 1, f)) { +- fprintf(stderr, +- "\nError: fread return a number of element different from the expected.\n"); +- opj_image_destroy(image); +- fclose(f); +- return NULL; +- } +- if (!fread(&r, 1, 1, f)) { +- fprintf(stderr, +- "\nError: fread return a number of element different from the expected.\n"); +- opj_image_destroy(image); +- fclose(f); +- return NULL; +- } +- +- image->comps[0].data[index] = r; +- image->comps[1].data[index] = g; +- image->comps[2].data[index] = b; +- index++; +- } +- } else if (numcomps == 4) { +- for (x = 0; x < image_width; x++) { +- unsigned char r, g, b, a; +- if (!fread(&b, 1, 1, f)) { +- fprintf(stderr, +- "\nError: fread return a number of element different from the expected.\n"); +- opj_image_destroy(image); +- fclose(f); +- return NULL; +- } +- if (!fread(&g, 1, 1, f)) { +- fprintf(stderr, +- "\nError: fread return a number of element different from the expected.\n"); +- opj_image_destroy(image); +- fclose(f); +- return NULL; +- } +- if (!fread(&r, 1, 1, f)) { +- fprintf(stderr, +- "\nError: fread return a number of element different from the expected.\n"); +- opj_image_destroy(image); +- fclose(f); +- return NULL; +- } +- if (!fread(&a, 1, 1, f)) { +- fprintf(stderr, +- "\nError: fread return a number of element different from the expected.\n"); +- opj_image_destroy(image); +- fclose(f); +- return NULL; +- } +- +- image->comps[0].data[index] = r; +- image->comps[1].data[index] = g; +- image->comps[2].data[index] = b; +- image->comps[3].data[index] = a; +- index++; +- } +- } else { +- fprintf(stderr, "Currently unsupported bit depth : %s\n", filename); +- } +- } +- fclose(f); +- return image; +-} +- +-int imagetotga(opj_image_t * image, const char *outfile) +-{ +- int width, height, bpp, x, y; +- OPJ_BOOL write_alpha; +- unsigned int i; +- int adjustR, adjustG = 0, adjustB = 0, fails; +- unsigned int alpha_channel; +- float r, g, b, a; +- unsigned char value; +- float scale; +- FILE *fdest; +- size_t res; +- fails = 1; ++ two = has_alpha = 0; ++ fails = 1; ++ ncomp = image->numcomps; ++ ++ while (*tmp) { ++ ++tmp; ++ } ++ tmp -= 2; ++ want_gray = (*tmp == 'g' || *tmp == 'G'); ++ ncomp = image->numcomps; ++ ++ if (want_gray) { ++ ncomp = 1; ++ } + ++ if ((force_split == 0) && ncomp >= 2 && are_comps_similar(image)) { + fdest = fopen(outfile, "wb"); ++ + if (!fdest) { +- fprintf(stderr, "ERROR -> failed to open %s for writing\n", outfile); +- return 1; ++ fprintf(stderr, "ERROR -> failed to open %s for writing\n", outfile); ++ return fails; + } +- +- for (i = 0; i < image->numcomps - 1; i++) { +- if ((image->comps[0].dx != image->comps[i + 1].dx) +- || (image->comps[0].dy != image->comps[i + 1].dy) +- || (image->comps[0].prec != image->comps[i + 1].prec) +- || (image->comps[0].sgnd != image->comps[i + 1].sgnd)) { +- fclose(fdest); +- fprintf(stderr, +- "Unable to create a tga file with such J2K image charateristics.\n"); +- return 1; +- } ++ two = (prec > 8); ++ triple = (ncomp > 2); ++ wr = (int)image->comps[0].w; ++ hr = (int)image->comps[0].h; ++ max = (1 << prec) - 1; ++ has_alpha = (ncomp == 4 || ncomp == 2); ++ ++ red = image->comps[0].data; ++ if (red == NULL) { ++ fprintf(stderr, "imagetopnm: planes[%d] == NULL.\n", 0); ++ fprintf(stderr, "\tAborting\n"); ++ fclose(fdest); ++ return fails; ++ } ++ ++ if (triple) { ++ green = image->comps[1].data; ++ blue = image->comps[2].data; ++ for (i = 1; i <= 2; i++) { ++ if (image->comps[i].data == NULL) { ++ fprintf(stderr, "imagetopnm: planes[%d] == NULL.\n", i); ++ fprintf(stderr, "\tAborting\n"); ++ fclose(fdest); ++ return fails; ++ } ++ } ++ } else { ++ green = blue = NULL; ++ } ++ ++ if (has_alpha) { ++ const char *tt = (triple ? "RGB_ALPHA" : "GRAYSCALE_ALPHA"); ++ ++ fprintf(fdest, ++ "P7\n# OpenJPEG-%s\nWIDTH %d\nHEIGHT %d\nDEPTH %u\n" ++ "MAXVAL %d\nTUPLTYPE %s\nENDHDR\n", ++ opj_version(), wr, hr, ncomp, max, tt); ++ alpha = image->comps[ncomp - 1].data; ++ adjustA = (image->comps[ncomp - 1].sgnd ++ ? 1 << (image->comps[ncomp - 1].prec - 1) ++ : 0); ++ } else { ++ fprintf(fdest, "P6\n# OpenJPEG-%s\n%d %d\n%d\n", opj_version(), wr, hr, ++ max); ++ adjustA = 0; + } +- +- width = (int)image->comps[0].w; +- height = (int)image->comps[0].h; +- +- /* Mono with alpha, or RGB with alpha. */ +- write_alpha = (image->numcomps == 2) || (image->numcomps == 4); +- +- /* Write TGA header */ +- bpp = write_alpha ? 32 : 24; +- +- if (!tga_writeheader(fdest, bpp, width, height, OPJ_TRUE)) { +- goto fin; +- } +- +- alpha_channel = image->numcomps - 1; +- +- scale = 255.0f / (float)((1 << image->comps[0].prec) - 1); +- + adjustR = (image->comps[0].sgnd ? 1 << (image->comps[0].prec - 1) : 0); +- if (image->numcomps >= 3) { +- adjustG = (image->comps[1].sgnd ? 1 << (image->comps[1].prec - 1) : 0); +- adjustB = (image->comps[2].sgnd ? 1 << (image->comps[2].prec - 1) : 0); +- } +- +- for (y = 0; y < height; y++) { +- unsigned int index = (unsigned int)(y * width); +- +- for (x = 0; x < width; x++, index++) { +- r = (float)(image->comps[0].data[index] + adjustR); + +- if (image->numcomps > 2) { +- g = (float)(image->comps[1].data[index] + adjustG); +- b = (float)(image->comps[2].data[index] + adjustB); +- } else { +- /* Greyscale ... */ +- g = r; +- b = r; +- } +- +- /* TGA format writes BGR ... */ +- if (b > 255.) { +- b = 255.; +- } else if (b < 0.) { +- b = 0.; +- } +- value = (unsigned char)(b * scale); +- res = fwrite(&value, 1, 1, fdest); +- +- if (res < 1) { +- fprintf(stderr, "failed to write 1 byte for %s\n", outfile); +- goto fin; +- } +- if (g > 255.) { +- g = 255.; +- } else if (g < 0.) { +- g = 0.; +- } +- value = (unsigned char)(g * scale); +- res = fwrite(&value, 1, 1, fdest); +- +- if (res < 1) { +- fprintf(stderr, "failed to write 1 byte for %s\n", outfile); +- goto fin; +- } +- if (r > 255.) { +- r = 255.; +- } else if (r < 0.) { +- r = 0.; +- } +- value = (unsigned char)(r * scale); +- res = fwrite(&value, 1, 1, fdest); +- +- if (res < 1) { +- fprintf(stderr, "failed to write 1 byte for %s\n", outfile); +- goto fin; +- } +- +- if (write_alpha) { +- a = (float)(image->comps[alpha_channel].data[index]); +- if (a > 255.) { +- a = 255.; +- } else if (a < 0.) { +- a = 0.; +- } +- value = (unsigned char)(a * scale); +- res = fwrite(&value, 1, 1, fdest); +- +- if (res < 1) { +- fprintf(stderr, "failed to write 1 byte for %s\n", outfile); +- goto fin; +- } +- } +- } +- } +- fails = 0; +-fin: +- fclose(fdest); +- +- return fails; +-} +- +-/* -->> -->> -->> -->> +- +-PGX IMAGE FORMAT +- +-<<-- <<-- <<-- <<-- */ +- +- +-static unsigned char readuchar(FILE * f) +-{ +- unsigned char c1; +- if (!fread(&c1, 1, 1, f)) { +- fprintf(stderr, +- "\nError: fread return a number of element different from the expected.\n"); +- return 0; +- } +- return c1; +-} +- +-static unsigned short readushort(FILE * f, int bigendian) +-{ +- unsigned char c1, c2; +- if (!fread(&c1, 1, 1, f)) { +- fprintf(stderr, +- "\nError: fread return a number of element different from the expected.\n"); +- return 0; +- } +- if (!fread(&c2, 1, 1, f)) { +- fprintf(stderr, +- "\nError: fread return a number of element different from the expected.\n"); +- return 0; +- } +- if (bigendian) { +- return (unsigned short)((c1 << 8) + c2); +- } else { +- return (unsigned short)((c2 << 8) + c1); +- } +-} +- +-static unsigned int readuint(FILE * f, int bigendian) +-{ +- unsigned char c1, c2, c3, c4; +- if (!fread(&c1, 1, 1, f)) { +- fprintf(stderr, +- "\nError: fread return a number of element different from the expected.\n"); +- return 0; +- } +- if (!fread(&c2, 1, 1, f)) { +- fprintf(stderr, +- "\nError: fread return a number of element different from the expected.\n"); +- return 0; +- } +- if (!fread(&c3, 1, 1, f)) { +- fprintf(stderr, +- "\nError: fread return a number of element different from the expected.\n"); +- return 0; +- } +- if (!fread(&c4, 1, 1, f)) { +- fprintf(stderr, +- "\nError: fread return a number of element different from the expected.\n"); +- return 0; +- } +- if (bigendian) { +- return (unsigned int)(c1 << 24) + (unsigned int)(c2 << 16) + (unsigned int)( +- c3 << 8) + c4; ++ if (triple) { ++ adjustG = (image->comps[1].sgnd ? 1 << (image->comps[1].prec - 1) : 0); ++ adjustB = (image->comps[2].sgnd ? 1 << (image->comps[2].prec - 1) : 0); + } else { +- return (unsigned int)(c4 << 24) + (unsigned int)(c3 << 16) + (unsigned int)( +- c2 << 8) + c1; +- } +-} +- +-opj_image_t* pgxtoimage(const char *filename, opj_cparameters_t *parameters) +-{ +- FILE *f = NULL; +- int w, h, prec; +- int i, numcomps, max; +- OPJ_COLOR_SPACE color_space; +- opj_image_cmptparm_t cmptparm; /* maximum of 1 component */ +- opj_image_t * image = NULL; +- int adjustS, ushift, dshift, force8; +- OPJ_UINT64 expected_file_size; +- +- char endian1, endian2, sign; +- char signtmp[32]; +- +- char temp[32]; +- int bigendian; +- opj_image_comp_t *comp = NULL; +- +- numcomps = 1; +- color_space = OPJ_CLRSPC_GRAY; +- +- memset(&cmptparm, 0, sizeof(opj_image_cmptparm_t)); +- +- max = 0; +- +- f = fopen(filename, "rb"); +- if (!f) { +- fprintf(stderr, "Failed to open %s for reading !\n", filename); +- return NULL; ++ adjustG = adjustB = 0; + } + +- fseek(f, 0, SEEK_SET); +- if (fscanf(f, "PG%31[ \t]%c%c%31[ \t+-]%d%31[ \t]%d%31[ \t]%d", temp, &endian1, +- &endian2, signtmp, &prec, temp, &w, temp, &h) != 9) { +- fclose(f); +- fprintf(stderr, +- "ERROR: Failed to read the right number of element from the fscanf() function!\n"); +- return NULL; +- } +- +- i = 0; +- sign = '+'; +- while (signtmp[i] != '\0') { +- if (signtmp[i] == '-') { +- sign = '-'; ++ for (i = 0; i < wr * hr; ++i) { ++ if (two) { ++ v = *red + adjustR; ++ ++red; ++ if (v > 65535) { ++ v = 65535; ++ } else if (v < 0) { ++ v = 0; + } +- i++; +- } + +- fgetc(f); +- if (endian1 == 'M' && endian2 == 'L') { +- bigendian = 1; +- } else if (endian2 == 'M' && endian1 == 'L') { +- bigendian = 0; +- } else { +- fclose(f); +- fprintf(stderr, "Bad pgx header, please check input file\n"); +- return NULL; +- } +- +- if (w < 1 || h < 1 || prec < 1 || prec > 31) { +- fclose(f); +- fprintf(stderr, "Bad pgx header, please check input file\n"); +- return NULL; +- } +- +- expected_file_size = +- (OPJ_UINT64)w * (OPJ_UINT64)h * (prec > 16 ? 4 : prec > 8 ? 2 : 1); +- if (expected_file_size > 10000000U) { +- char ch; +- long curpos = ftell(f); +- if (expected_file_size > (OPJ_UINT64)INT_MAX) { +- expected_file_size = (OPJ_UINT64)INT_MAX; +- } +- fseek(f, (long)expected_file_size - 1, SEEK_SET); +- if (fread(&ch, 1, 1, f) != 1) { +- fprintf(stderr, "File too short\n"); +- fclose(f); +- return NULL; +- } +- fseek(f, curpos, SEEK_SET); +- } +- +- /* initialize image component */ +- +- cmptparm.x0 = (OPJ_UINT32)parameters->image_offset_x0; +- cmptparm.y0 = (OPJ_UINT32)parameters->image_offset_y0; +- cmptparm.w = !cmptparm.x0 ? (OPJ_UINT32)((w - 1) * parameters->subsampling_dx + +- 1) : cmptparm.x0 + (OPJ_UINT32)(w - 1) * (OPJ_UINT32)parameters->subsampling_dx +- + 1; +- cmptparm.h = !cmptparm.y0 ? (OPJ_UINT32)((h - 1) * parameters->subsampling_dy + +- 1) : cmptparm.y0 + (OPJ_UINT32)(h - 1) * (OPJ_UINT32)parameters->subsampling_dy +- + 1; +- +- if (sign == '-') { +- cmptparm.sgnd = 1; +- } else { +- cmptparm.sgnd = 0; +- } +- if (prec < 8) { +- force8 = 1; +- ushift = 8 - prec; +- dshift = prec - ushift; +- if (cmptparm.sgnd) { +- adjustS = (1 << (prec - 1)); +- } else { +- adjustS = 0; +- } +- cmptparm.sgnd = 0; +- prec = 8; +- } else { +- ushift = dshift = force8 = adjustS = 0; +- } +- +- cmptparm.prec = (OPJ_UINT32)prec; +- cmptparm.bpp = (OPJ_UINT32)prec; +- cmptparm.dx = (OPJ_UINT32)parameters->subsampling_dx; +- cmptparm.dy = (OPJ_UINT32)parameters->subsampling_dy; +- +- /* create the image */ +- image = opj_image_create((OPJ_UINT32)numcomps, &cmptparm, color_space); +- if (!image) { +- fclose(f); +- return NULL; +- } +- /* set image offset and reference grid */ +- image->x0 = cmptparm.x0; +- image->y0 = cmptparm.x0; +- image->x1 = cmptparm.w; +- image->y1 = cmptparm.h; +- +- /* set image data */ +- +- comp = &image->comps[0]; +- +- for (i = 0; i < w * h; i++) { +- int v; +- if (force8) { +- v = readuchar(f) + adjustS; +- v = (v << ushift) + (v >> dshift); +- comp->data[i] = (unsigned char)v; +- +- if (v > max) { +- max = v; +- } +- +- continue; +- } +- if (comp->prec == 8) { +- if (!comp->sgnd) { +- v = readuchar(f); +- } else { +- v = (char) readuchar(f); +- } +- } else if (comp->prec <= 16) { +- if (!comp->sgnd) { +- v = readushort(f, bigendian); +- } else { +- v = (short) readushort(f, bigendian); +- } +- } else { +- if (!comp->sgnd) { +- v = (int)readuint(f, bigendian); +- } else { +- v = (int) readuint(f, bigendian); +- } +- } +- if (v > max) { +- max = v; +- } +- comp->data[i] = v; +- } +- fclose(f); +- comp->bpp = (OPJ_UINT32)int_floorlog2(max) + 1; +- +- return image; +-} +- +-#define CLAMP(x,a,b) ((x) < (a) ? (a) : ((x) > (b) ? (b) : (x))) +- +-static INLINE int clamp(const int value, const int prec, const int sgnd) +-{ +- if (sgnd) { +- if (prec <= 8) { +- return CLAMP(value, -128, 127); +- } else if (prec <= 16) { +- return CLAMP(value, -32768, 32767); +- } else { +- return CLAMP(value, -2147483647 - 1, 2147483647); +- } +- } else { +- if (prec <= 8) { +- return CLAMP(value, 0, 255); +- } else if (prec <= 16) { +- return CLAMP(value, 0, 65535); +- } else { +- return value; /*CLAMP(value,0,4294967295);*/ +- } +- } +-} +- +-int imagetopgx(opj_image_t * image, const char *outfile) +-{ +- int w, h; +- int i, j, fails = 1; +- unsigned int compno; +- FILE *fdest = NULL; +- +- for (compno = 0; compno < image->numcomps; compno++) { +- opj_image_comp_t *comp = &image->comps[compno]; +- char bname[256]; /* buffer for name */ +- char *name = bname; /* pointer */ +- int nbytes = 0; +- size_t res; +- const size_t olen = strlen(outfile); +- const size_t dotpos = olen - 4; +- const size_t total = dotpos + 1 + 1 + 4; /* '-' + '[1-3]' + '.pgx' */ +- +- if (outfile[dotpos] != '.') { +- /* `pgx` was recognized but there is no dot at expected position */ +- fprintf(stderr, "ERROR -> Impossible happen."); +- goto fin; +- } +- if (total > 256) { +- name = (char*)malloc(total + 1); +- if (name == NULL) { +- fprintf(stderr, "imagetopgx: memory out\n"); +- goto fin; +- } +- } +- strncpy(name, outfile, dotpos); +- sprintf(name + dotpos, "_%u.pgx", compno); +- fdest = fopen(name, "wb"); +- /* don't need name anymore */ +- +- if (!fdest) { +- +- fprintf(stderr, "ERROR -> failed to open %s for writing\n", name); +- if (total > 256) { +- free(name); +- } +- goto fin; +- } +- +- w = (int)image->comps[compno].w; +- h = (int)image->comps[compno].h; +- +- fprintf(fdest, "PG ML %c %d %d %d\n", comp->sgnd ? '-' : '+', comp->prec, +- w, h); +- +- if (comp->prec <= 8) { +- nbytes = 1; +- } else if (comp->prec <= 16) { +- nbytes = 2; +- } else { +- nbytes = 4; +- } +- +- if (nbytes == 1) { +- unsigned char* line_buffer = malloc((size_t)w); +- if (line_buffer == NULL) { +- fprintf(stderr, "Out of memory"); +- if (total > 256) { +- free(name); +- } +- goto fin; +- } +- for (j = 0; j < h; j++) { +- if (comp->prec == 8 && comp->sgnd == 0) { +- for (i = 0; i < w; i++) { +- line_buffer[i] = (unsigned char)CLAMP(image->comps[compno].data[j * w + i], 0, +- 255); +- } +- } else { +- for (i = 0; i < w; i++) { +- line_buffer[i] = (unsigned char) +- clamp(image->comps[compno].data[j * w + i], +- (int)comp->prec, (int)comp->sgnd); +- } +- } +- res = fwrite(line_buffer, 1, (size_t)w, fdest); +- if (res != (size_t)w) { +- fprintf(stderr, "failed to write %d bytes for %s\n", w, name); +- if (total > 256) { +- free(name); +- } +- free(line_buffer); +- goto fin; +- } +- } +- free(line_buffer); +- } else { +- +- for (i = 0; i < w * h; i++) { +- /* FIXME: clamp func is being called within a loop */ +- const int val = clamp(image->comps[compno].data[i], +- (int)comp->prec, (int)comp->sgnd); +- +- for (j = nbytes - 1; j >= 0; j--) { +- int v = (int)(val >> (j * 8)); +- unsigned char byte = (unsigned char)v; +- res = fwrite(&byte, 1, 1, fdest); +- +- if (res < 1) { +- fprintf(stderr, "failed to write 1 byte for %s\n", name); +- if (total > 256) { +- free(name); +- } +- goto fin; +- } +- } +- } +- } +- +- if (total > 256) { +- free(name); +- } +- fclose(fdest); +- fdest = NULL; +- } +- fails = 0; +-fin: +- if (fdest) { +- fclose(fdest); +- } +- +- return fails; +-} +- +-/* -->> -->> -->> -->> +- +-PNM IMAGE FORMAT +- +-<<-- <<-- <<-- <<-- */ +- +-struct pnm_header { +- int width, height, maxval, depth, format; +- char rgb, rgba, gray, graya, bw; +- char ok; +-}; +- +-static char *skip_white(char *s) +-{ +- if (s != NULL) { +- while (*s) { +- if (*s == '\n' || *s == '\r') { +- return NULL; +- } +- if (isspace(*s)) { +- ++s; +- continue; +- } +- return s; +- } +- } +- return NULL; +-} +- +-static char *skip_int(char *start, int *out_n) +-{ +- char *s; +- char c; +- +- *out_n = 0; +- +- s = skip_white(start); +- if (s == NULL) { +- return NULL; +- } +- start = s; +- +- while (*s) { +- if (!isdigit(*s)) { +- break; +- } +- ++s; +- } +- c = *s; +- *s = 0; +- *out_n = atoi(start); +- *s = c; +- return s; +-} +- +-static char *skip_idf(char *start, char out_idf[256]) +-{ +- char *s; +- char c; +- +- s = skip_white(start); +- if (s == NULL) { +- return NULL; +- } +- start = s; +- +- while (*s) { +- if (isalpha(*s) || *s == '_') { +- ++s; +- continue; +- } +- break; +- } +- c = *s; +- *s = 0; +- strncpy(out_idf, start, 255); +- *s = c; +- return s; +-} +- +-static void read_pnm_header(FILE *reader, struct pnm_header *ph) +-{ +- int format, end, ttype; +- char idf[256], type[256]; +- char line[256]; +- +- if (fgets(line, 250, reader) == NULL) { +- fprintf(stderr, "\nWARNING: fgets return a NULL value"); +- return; +- } +- +- if (line[0] != 'P') { +- fprintf(stderr, "read_pnm_header:PNM:magic P missing\n"); +- return; +- } +- format = atoi(line + 1); +- if (format < 1 || format > 7) { +- fprintf(stderr, "read_pnm_header:magic format %d invalid\n", format); +- return; +- } +- ph->format = format; +- ttype = end = 0; +- +- while (fgets(line, 250, reader)) { +- char *s; +- int allow_null = 0; +- +- if (*line == '#') { +- continue; +- } +- +- s = line; +- +- if (format == 7) { +- s = skip_idf(s, idf); +- +- if (s == NULL || *s == 0) { +- return; +- } +- +- if (strcmp(idf, "ENDHDR") == 0) { +- end = 1; +- break; +- } +- if (strcmp(idf, "WIDTH") == 0) { +- s = skip_int(s, &ph->width); +- if (s == NULL || *s == 0) { +- return; +- } +- +- continue; +- } +- if (strcmp(idf, "HEIGHT") == 0) { +- s = skip_int(s, &ph->height); +- if (s == NULL || *s == 0) { +- return; +- } +- +- continue; +- } +- if (strcmp(idf, "DEPTH") == 0) { +- s = skip_int(s, &ph->depth); +- if (s == NULL || *s == 0) { +- return; +- } +- +- continue; +- } +- if (strcmp(idf, "MAXVAL") == 0) { +- s = skip_int(s, &ph->maxval); +- if (s == NULL || *s == 0) { +- return; +- } +- +- continue; +- } +- if (strcmp(idf, "TUPLTYPE") == 0) { +- s = skip_idf(s, type); +- if (s == NULL || *s == 0) { +- return; +- } +- +- if (strcmp(type, "BLACKANDWHITE") == 0) { +- ph->bw = 1; +- ttype = 1; +- continue; +- } +- if (strcmp(type, "GRAYSCALE") == 0) { +- ph->gray = 1; +- ttype = 1; +- continue; +- } +- if (strcmp(type, "GRAYSCALE_ALPHA") == 0) { +- ph->graya = 1; +- ttype = 1; +- continue; +- } +- if (strcmp(type, "RGB") == 0) { +- ph->rgb = 1; +- ttype = 1; +- continue; +- } +- if (strcmp(type, "RGB_ALPHA") == 0) { +- ph->rgba = 1; +- ttype = 1; +- continue; +- } +- fprintf(stderr, "read_pnm_header:unknown P7 TUPLTYPE %s\n", type); +- return; +- } +- fprintf(stderr, "read_pnm_header:unknown P7 idf %s\n", idf); +- return; +- } /* if(format == 7) */ +- +- /* Here format is in range [1,6] */ +- if (ph->width == 0) { +- s = skip_int(s, &ph->width); +- if ((s == NULL) || (*s == 0) || (ph->width < 1)) { +- return; +- } +- allow_null = 1; +- } +- if (ph->height == 0) { +- s = skip_int(s, &ph->height); +- if ((s == NULL) && allow_null) { +- continue; +- } +- if ((s == NULL) || (*s == 0) || (ph->height < 1)) { +- return; +- } +- if (format == 1 || format == 4) { +- break; +- } +- allow_null = 1; +- } +- /* here, format is in P2, P3, P5, P6 */ +- s = skip_int(s, &ph->maxval); +- if ((s == NULL) && allow_null) { +- continue; +- } +- if ((s == NULL) || (*s == 0)) { +- return; +- } +- break; +- }/* while(fgets( ) */ +- if (format == 2 || format == 3 || format > 4) { +- if (ph->maxval < 1 || ph->maxval > 65535) { +- return; +- } +- } +- if (ph->width < 1 || ph->height < 1) { +- return; +- } +- +- if (format == 7) { +- if (!end) { +- fprintf(stderr, "read_pnm_header:P7 without ENDHDR\n"); +- return; +- } +- if (ph->depth < 1 || ph->depth > 4) { +- return; +- } +- +- if (ttype) { +- ph->ok = 1; +- } +- } else { +- ph->ok = 1; +- if (format == 1 || format == 4) { +- ph->maxval = 255; +- } +- } +-} +- +-static int has_prec(int val) +-{ +- if (val < 2) { +- return 1; +- } +- if (val < 4) { +- return 2; +- } +- if (val < 8) { +- return 3; +- } +- if (val < 16) { +- return 4; +- } +- if (val < 32) { +- return 5; +- } +- if (val < 64) { +- return 6; +- } +- if (val < 128) { +- return 7; +- } +- if (val < 256) { +- return 8; +- } +- if (val < 512) { +- return 9; +- } +- if (val < 1024) { +- return 10; +- } +- if (val < 2048) { +- return 11; +- } +- if (val < 4096) { +- return 12; +- } +- if (val < 8192) { +- return 13; +- } +- if (val < 16384) { +- return 14; +- } +- if (val < 32768) { +- return 15; +- } +- return 16; +-} +- +-opj_image_t* pnmtoimage(const char *filename, opj_cparameters_t *parameters) +-{ +- int subsampling_dx = parameters->subsampling_dx; +- int subsampling_dy = parameters->subsampling_dy; +- +- FILE *fp = NULL; +- int i, compno, numcomps, w, h, prec, format; +- OPJ_COLOR_SPACE color_space; +- opj_image_cmptparm_t cmptparm[4]; /* RGBA: max. 4 components */ +- opj_image_t * image = NULL; +- struct pnm_header header_info; +- +- if ((fp = fopen(filename, "rb")) == NULL) { +- fprintf(stderr, "pnmtoimage:Failed to open %s for reading!\n", filename); +- return NULL; +- } +- memset(&header_info, 0, sizeof(struct pnm_header)); +- +- read_pnm_header(fp, &header_info); +- +- if (!header_info.ok) { +- fclose(fp); +- return NULL; +- } +- +- if (header_info.width == 0 +- || header_info.height == 0 +- || (header_info.format == 7 && header_info.depth == 0)) { +- fclose(fp); +- return NULL; +- } +- +- /* This limitation could be removed by making sure to use size_t below */ +- if (header_info.height != 0 && +- header_info.width > INT_MAX / header_info.height) { +- fprintf(stderr, "pnmtoimage:Image %dx%d too big!\n", +- header_info.width, header_info.height); +- fclose(fp); +- return NULL; +- } +- +- format = header_info.format; +- +- switch (format) { +- case 1: /* ascii bitmap */ +- case 4: /* raw bitmap */ +- numcomps = 1; +- break; +- +- case 2: /* ascii greymap */ +- case 5: /* raw greymap */ +- numcomps = 1; +- break; +- +- case 3: /* ascii pixmap */ +- case 6: /* raw pixmap */ +- numcomps = 3; +- break; +- +- case 7: /* arbitrary map */ +- numcomps = header_info.depth; +- break; +- +- default: +- fclose(fp); +- return NULL; +- } +- if (numcomps < 3) { +- color_space = OPJ_CLRSPC_GRAY; /* GRAY, GRAYA */ +- } else { +- color_space = OPJ_CLRSPC_SRGB; /* RGB, RGBA */ +- } +- +- prec = has_prec(header_info.maxval); +- +- if (prec < 8) { +- prec = 8; +- } +- +- w = header_info.width; +- h = header_info.height; +- subsampling_dx = parameters->subsampling_dx; +- subsampling_dy = parameters->subsampling_dy; +- +- memset(&cmptparm[0], 0, (size_t)numcomps * sizeof(opj_image_cmptparm_t)); +- +- for (i = 0; i < numcomps; i++) { +- cmptparm[i].prec = (OPJ_UINT32)prec; +- cmptparm[i].bpp = (OPJ_UINT32)prec; +- cmptparm[i].sgnd = 0; +- cmptparm[i].dx = (OPJ_UINT32)subsampling_dx; +- cmptparm[i].dy = (OPJ_UINT32)subsampling_dy; +- cmptparm[i].w = (OPJ_UINT32)w; +- cmptparm[i].h = (OPJ_UINT32)h; +- } +- image = opj_image_create((OPJ_UINT32)numcomps, &cmptparm[0], color_space); +- +- if (!image) { +- fclose(fp); +- return NULL; +- } +- +- /* set image offset and reference grid */ +- image->x0 = (OPJ_UINT32)parameters->image_offset_x0; +- image->y0 = (OPJ_UINT32)parameters->image_offset_y0; +- image->x1 = (OPJ_UINT32)(parameters->image_offset_x0 + (w - 1) * subsampling_dx +- + 1); +- image->y1 = (OPJ_UINT32)(parameters->image_offset_y0 + (h - 1) * subsampling_dy +- + 1); +- +- if ((format == 2) || (format == 3)) { /* ascii pixmap */ +- unsigned int index; +- +- for (i = 0; i < w * h; i++) { +- for (compno = 0; compno < numcomps; compno++) { +- index = 0; +- if (fscanf(fp, "%u", &index) != 1) { +- fprintf(stderr, "Missing data. Quitting.\n"); +- opj_image_destroy(image); +- fclose(fp); +- return NULL; +- } +- +- image->comps[compno].data[i] = (OPJ_INT32)(index * 255) / header_info.maxval; +- } +- } +- } else if ((format == 5) +- || (format == 6) +- || ((format == 7) +- && (header_info.gray || header_info.graya +- || header_info.rgb || header_info.rgba))) { /* binary pixmap */ +- unsigned char c0, c1, one; +- +- one = (prec < 9); +- +- for (i = 0; i < w * h; i++) { +- for (compno = 0; compno < numcomps; compno++) { +- if (!fread(&c0, 1, 1, fp)) { +- fprintf(stderr, "Missing data. Quitting.\n"); +- opj_image_destroy(image); +- fclose(fp); +- return NULL; +- } +- if (one) { +- image->comps[compno].data[i] = c0; +- } else { +- if (!fread(&c1, 1, 1, fp)) { +- fprintf(stderr, "Missing data. Quitting.\n"); +- opj_image_destroy(image); +- fclose(fp); +- return NULL; +- } +- /* netpbm: */ +- image->comps[compno].data[i] = ((c0 << 8) | c1); +- } +- } +- } +- } else if (format == 1) { /* ascii bitmap */ +- for (i = 0; i < w * h; i++) { +- unsigned int index; +- +- if (fscanf(fp, "%u", &index) != 1) { +- fprintf(stderr, "Missing data. Quitting.\n"); +- opj_image_destroy(image); +- fclose(fp); +- return NULL; +- } +- +- image->comps[0].data[i] = (index ? 0 : 255); +- } +- } else if (format == 4) { +- int x, y, bit; +- int uc; +- +- i = 0; +- for (y = 0; y < h; ++y) { +- bit = -1; +- uc = 0; +- +- for (x = 0; x < w; ++x) { +- if (bit == -1) { +- bit = 7; +- uc = getc(fp); +- if (uc == EOF) { +- fprintf(stderr, "Missing data. Quitting.\n"); +- opj_image_destroy(image); +- fclose(fp); +- return NULL; +- } +- } +- image->comps[0].data[i] = ((((unsigned char)uc >> bit) & 1) ? 0 : 255); +- --bit; +- ++i; +- } +- } +- } else if ((format == 7 && header_info.bw)) { /*MONO*/ +- unsigned char uc; +- +- for (i = 0; i < w * h; ++i) { +- if (!fread(&uc, 1, 1, fp)) { +- fprintf(stderr, "Missing data. Quitting.\n"); +- opj_image_destroy(image); +- fclose(fp); +- return NULL; +- } +- image->comps[0].data[i] = (uc & 1) ? 0 : 255; +- } +- } +- fclose(fp); +- +- return image; +-}/* pnmtoimage() */ +- +-static int are_comps_similar(opj_image_t * image) +-{ +- unsigned int i; +- for (i = 1; i < image->numcomps; i++) { +- if (image->comps[0].dx != image->comps[i].dx || +- image->comps[0].dy != image->comps[i].dy || +- (i <= 2 && +- (image->comps[0].prec != image->comps[i].prec || +- image->comps[0].sgnd != image->comps[i].sgnd))) { +- return OPJ_FALSE; +- } +- } +- return OPJ_TRUE; +-} +- +- +-int imagetopnm(opj_image_t * image, const char *outfile, int force_split) +-{ +- int *red, *green, *blue, *alpha; +- int wr, hr, max; +- int i; +- unsigned int compno, ncomp; +- int adjustR, adjustG, adjustB, adjustA; +- int fails, two, want_gray, has_alpha, triple; +- int prec, v; +- FILE *fdest = NULL; +- const char *tmp = outfile; +- char *destname; +- +- alpha = NULL; +- +- if ((prec = (int)image->comps[0].prec) > 16) { +- fprintf(stderr, "%s:%d:imagetopnm\n\tprecision %d is larger than 16" +- "\n\t: refused.\n", __FILE__, __LINE__, prec); +- return 1; +- } +- two = has_alpha = 0; +- fails = 1; +- ncomp = image->numcomps; +- +- while (*tmp) { +- ++tmp; +- } +- tmp -= 2; +- want_gray = (*tmp == 'g' || *tmp == 'G'); +- ncomp = image->numcomps; +- +- if (want_gray) { +- ncomp = 1; +- } +- +- if ((force_split == 0) && ncomp >= 2 && +- are_comps_similar(image)) { +- fdest = fopen(outfile, "wb"); +- +- if (!fdest) { +- fprintf(stderr, "ERROR -> failed to open %s for writing\n", outfile); +- return fails; +- } +- two = (prec > 8); +- triple = (ncomp > 2); +- wr = (int)image->comps[0].w; +- hr = (int)image->comps[0].h; +- max = (1 << prec) - 1; +- has_alpha = (ncomp == 4 || ncomp == 2); +- +- red = image->comps[0].data; +- if (red == NULL) { +- fprintf(stderr, +- "imagetopnm: planes[%d] == NULL.\n", 0); +- fprintf(stderr, "\tAborting\n"); +- fclose(fdest); +- return fails; +- } +- +- if (triple) { +- green = image->comps[1].data; +- blue = image->comps[2].data; +- for (i = 1; i <= 2; i++) { +- if (image->comps[i].data == NULL) { +- fprintf(stderr, +- "imagetopnm: planes[%d] == NULL.\n", i); +- fprintf(stderr, "\tAborting\n"); +- fclose(fdest); +- return fails; +- } +- } +- } else { +- green = blue = NULL; +- } +- +- if (has_alpha) { +- const char *tt = (triple ? "RGB_ALPHA" : "GRAYSCALE_ALPHA"); +- +- fprintf(fdest, "P7\n# OpenJPEG-%s\nWIDTH %d\nHEIGHT %d\nDEPTH %u\n" +- "MAXVAL %d\nTUPLTYPE %s\nENDHDR\n", opj_version(), +- wr, hr, ncomp, max, tt); +- alpha = image->comps[ncomp - 1].data; +- adjustA = (image->comps[ncomp - 1].sgnd ? +- 1 << (image->comps[ncomp - 1].prec - 1) : 0); +- } else { +- fprintf(fdest, "P6\n# OpenJPEG-%s\n%d %d\n%d\n", +- opj_version(), wr, hr, max); +- adjustA = 0; +- } +- adjustR = (image->comps[0].sgnd ? 1 << (image->comps[0].prec - 1) : 0); ++ /* netpbm: */ ++ fprintf(fdest, "%c%c", (unsigned char)(v >> 8), (unsigned char)v); + + if (triple) { +- adjustG = (image->comps[1].sgnd ? 1 << (image->comps[1].prec - 1) : 0); +- adjustB = (image->comps[2].sgnd ? 1 << (image->comps[2].prec - 1) : 0); +- } else { +- adjustG = adjustB = 0; +- } +- +- for (i = 0; i < wr * hr; ++i) { +- if (two) { +- v = *red + adjustR; +- ++red; +- if (v > 65535) { +- v = 65535; +- } else if (v < 0) { +- v = 0; +- } +- +- /* netpbm: */ +- fprintf(fdest, "%c%c", (unsigned char)(v >> 8), (unsigned char)v); +- +- if (triple) { +- v = *green + adjustG; +- ++green; +- if (v > 65535) { +- v = 65535; +- } else if (v < 0) { +- v = 0; +- } +- +- /* netpbm: */ +- fprintf(fdest, "%c%c", (unsigned char)(v >> 8), (unsigned char)v); +- +- v = *blue + adjustB; +- ++blue; +- if (v > 65535) { +- v = 65535; +- } else if (v < 0) { +- v = 0; +- } +- +- /* netpbm: */ +- fprintf(fdest, "%c%c", (unsigned char)(v >> 8), (unsigned char)v); +- +- }/* if(triple) */ +- +- if (has_alpha) { +- v = *alpha + adjustA; +- ++alpha; +- if (v > 65535) { +- v = 65535; +- } else if (v < 0) { +- v = 0; +- } +- +- /* netpbm: */ +- fprintf(fdest, "%c%c", (unsigned char)(v >> 8), (unsigned char)v); +- } +- continue; +- +- } /* if(two) */ +- +- /* prec <= 8: */ +- v = *red++; +- if (v > 255) { +- v = 255; +- } else if (v < 0) { +- v = 0; +- } +- +- fprintf(fdest, "%c", (unsigned char)v); +- if (triple) { +- v = *green++; +- if (v > 255) { +- v = 255; +- } else if (v < 0) { +- v = 0; +- } +- +- fprintf(fdest, "%c", (unsigned char)v); +- v = *blue++; +- if (v > 255) { +- v = 255; +- } else if (v < 0) { +- v = 0; +- } +- +- fprintf(fdest, "%c", (unsigned char)v); +- } +- if (has_alpha) { +- v = *alpha++; +- if (v > 255) { +- v = 255; +- } else if (v < 0) { +- v = 0; +- } +- +- fprintf(fdest, "%c", (unsigned char)v); +- } +- } /* for(i */ ++ v = *green + adjustG; ++ ++green; ++ if (v > 65535) { ++ v = 65535; ++ } else if (v < 0) { ++ v = 0; ++ } ++ ++ /* netpbm: */ ++ fprintf(fdest, "%c%c", (unsigned char)(v >> 8), (unsigned char)v); ++ ++ v = *blue + adjustB; ++ ++blue; ++ if (v > 65535) { ++ v = 65535; ++ } else if (v < 0) { ++ v = 0; ++ } + +- fclose(fdest); +- return 0; +- } ++ /* netpbm: */ ++ fprintf(fdest, "%c%c", (unsigned char)(v >> 8), (unsigned char)v); + +- /* YUV or MONO: */ ++ } /* if(triple) */ + +- if (image->numcomps > ncomp) { +- fprintf(stderr, "WARNING -> [PGM file] Only the first component\n"); +- fprintf(stderr, " is written to the file\n"); +- } +- destname = (char*)malloc(strlen(outfile) + 8); +- if (destname == NULL) { +- fprintf(stderr, "imagetopnm: memory out\n"); +- return 1; +- } +- for (compno = 0; compno < ncomp; compno++) { +- if (ncomp > 1) { +- /*sprintf(destname, "%d.%s", compno, outfile);*/ +- const size_t olen = strlen(outfile); +- const size_t dotpos = olen - 4; +- +- strncpy(destname, outfile, dotpos); +- sprintf(destname + dotpos, "_%u.pgm", compno); +- } else { +- sprintf(destname, "%s", outfile); +- } +- +- fdest = fopen(destname, "wb"); +- if (!fdest) { +- fprintf(stderr, "ERROR -> failed to open %s for writing\n", destname); +- free(destname); +- return 1; +- } +- wr = (int)image->comps[compno].w; +- hr = (int)image->comps[compno].h; +- prec = (int)image->comps[compno].prec; +- max = (1 << prec) - 1; +- +- fprintf(fdest, "P5\n#OpenJPEG-%s\n%d %d\n%d\n", +- opj_version(), wr, hr, max); +- +- red = image->comps[compno].data; +- if (!red) { +- fclose(fdest); +- continue; +- } +- +- adjustR = +- (image->comps[compno].sgnd ? 1 << (image->comps[compno].prec - 1) : 0); +- +- if (prec > 8) { +- for (i = 0; i < wr * hr; i++) { +- v = *red + adjustR; +- ++red; +- if (v > 65535) { +- v = 65535; +- } else if (v < 0) { +- v = 0; +- } +- +- /* netpbm: */ +- fprintf(fdest, "%c%c", (unsigned char)(v >> 8), (unsigned char)v); +- +- if (has_alpha) { +- v = *alpha++; +- if (v > 65535) { +- v = 65535; +- } else if (v < 0) { +- v = 0; +- } +- +- /* netpbm: */ +- fprintf(fdest, "%c%c", (unsigned char)(v >> 8), (unsigned char)v); +- } +- }/* for(i */ +- } else { /* prec <= 8 */ +- for (i = 0; i < wr * hr; ++i) { +- v = *red + adjustR; +- ++red; +- if (v > 255) { +- v = 255; +- } else if (v < 0) { +- v = 0; +- } +- +- fprintf(fdest, "%c", (unsigned char)v); +- } +- } +- fclose(fdest); +- } /* for (compno */ +- free(destname); ++ if (has_alpha) { ++ v = *alpha + adjustA; ++ ++alpha; ++ if (v > 65535) { ++ v = 65535; ++ } else if (v < 0) { ++ v = 0; ++ } ++ ++ /* netpbm: */ ++ fprintf(fdest, "%c%c", (unsigned char)(v >> 8), (unsigned char)v); ++ } ++ continue; ++ ++ } /* if(two) */ ++ ++ /* prec <= 8: */ ++ v = *red++; ++ if (v > 255) { ++ v = 255; ++ } else if (v < 0) { ++ v = 0; ++ } ++ ++ fprintf(fdest, "%c", (unsigned char)v); ++ if (triple) { ++ v = *green++; ++ if (v > 255) { ++ v = 255; ++ } else if (v < 0) { ++ v = 0; ++ } ++ ++ fprintf(fdest, "%c", (unsigned char)v); ++ v = *blue++; ++ if (v > 255) { ++ v = 255; ++ } else if (v < 0) { ++ v = 0; ++ } ++ ++ fprintf(fdest, "%c", (unsigned char)v); ++ } ++ if (has_alpha) { ++ v = *alpha++; ++ if (v > 255) { ++ v = 255; ++ } else if (v < 0) { ++ v = 0; ++ } ++ ++ fprintf(fdest, "%c", (unsigned char)v); ++ } ++ } /* for(i */ + ++ fclose(fdest); + return 0; +-}/* imagetopnm() */ +- +-/* -->> -->> -->> -->> ++ } + +- RAW IMAGE FORMAT ++ /* YUV or MONO: */ + +- <<-- <<-- <<-- <<-- */ +-static opj_image_t* rawtoimage_common(const char *filename, +- opj_cparameters_t *parameters, raw_cparameters_t *raw_cp, OPJ_BOOL big_endian) +-{ +- int subsampling_dx = parameters->subsampling_dx; +- int subsampling_dy = parameters->subsampling_dy; +- +- FILE *f = NULL; +- int i, compno, numcomps, w, h; +- OPJ_COLOR_SPACE color_space; +- opj_image_cmptparm_t *cmptparm; +- opj_image_t * image = NULL; +- unsigned short ch; +- +- if ((!(raw_cp->rawWidth & raw_cp->rawHeight & raw_cp->rawComp & +- raw_cp->rawBitDepth)) == 0) { +- fprintf(stderr, "\nError: invalid raw image parameters\n"); +- fprintf(stderr, "Please use the Format option -F:\n"); +- fprintf(stderr, +- "-F ,,,,{s,u}@x:...:x\n"); +- fprintf(stderr, +- "If subsampling is omitted, 1x1 is assumed for all components\n"); +- fprintf(stderr, +- "Example: -i image.raw -o image.j2k -F 512,512,3,8,u@1x1:2x2:2x2\n"); +- fprintf(stderr, " for raw 512x512 image with 4:2:0 subsampling\n"); +- fprintf(stderr, "Aborting.\n"); +- return NULL; +- } +- +- f = fopen(filename, "rb"); +- if (!f) { +- fprintf(stderr, "Failed to open %s for reading !!\n", filename); +- fprintf(stderr, "Aborting\n"); +- return NULL; +- } +- numcomps = raw_cp->rawComp; ++ if (image->numcomps > ncomp) { ++ fprintf(stderr, "WARNING -> [PGM file] Only the first component\n"); ++ fprintf(stderr, " is written to the file\n"); ++ } ++ destname = (char *)malloc(strlen(outfile) + 8); ++ if (destname == NULL) { ++ fprintf(stderr, "imagetopnm: memory out\n"); ++ return 1; ++ } ++ for (compno = 0; compno < ncomp; compno++) { ++ if (ncomp > 1) { ++ /*sprintf(destname, "%d.%s", compno, outfile);*/ ++ const size_t olen = strlen(outfile); ++ const size_t dotpos = olen - 4; + +- /* FIXME ADE at this point, tcp_mct has not been properly set in calling function */ +- if (numcomps == 1) { +- color_space = OPJ_CLRSPC_GRAY; +- } else if ((numcomps >= 3) && (parameters->tcp_mct == 0)) { +- color_space = OPJ_CLRSPC_SYCC; +- } else if ((numcomps >= 3) && (parameters->tcp_mct != 2)) { +- color_space = OPJ_CLRSPC_SRGB; ++ strncpy(destname, outfile, dotpos); ++ sprintf(destname + dotpos, "_%u.pgm", compno); + } else { +- color_space = OPJ_CLRSPC_UNKNOWN; +- } +- w = raw_cp->rawWidth; +- h = raw_cp->rawHeight; +- cmptparm = (opj_image_cmptparm_t*) calloc((OPJ_UINT32)numcomps, +- sizeof(opj_image_cmptparm_t)); +- if (!cmptparm) { +- fprintf(stderr, "Failed to allocate image components parameters !!\n"); +- fprintf(stderr, "Aborting\n"); +- fclose(f); +- return NULL; +- } +- /* initialize image components */ +- for (i = 0; i < numcomps; i++) { +- cmptparm[i].prec = (OPJ_UINT32)raw_cp->rawBitDepth; +- cmptparm[i].bpp = (OPJ_UINT32)raw_cp->rawBitDepth; +- cmptparm[i].sgnd = (OPJ_UINT32)raw_cp->rawSigned; +- cmptparm[i].dx = (OPJ_UINT32)(subsampling_dx * raw_cp->rawComps[i].dx); +- cmptparm[i].dy = (OPJ_UINT32)(subsampling_dy * raw_cp->rawComps[i].dy); +- cmptparm[i].w = (OPJ_UINT32)w; +- cmptparm[i].h = (OPJ_UINT32)h; +- } +- /* create the image */ +- image = opj_image_create((OPJ_UINT32)numcomps, &cmptparm[0], color_space); +- free(cmptparm); +- if (!image) { +- fclose(f); +- return NULL; +- } +- /* set image offset and reference grid */ +- image->x0 = (OPJ_UINT32)parameters->image_offset_x0; +- image->y0 = (OPJ_UINT32)parameters->image_offset_y0; +- image->x1 = (OPJ_UINT32)parameters->image_offset_x0 + (OPJ_UINT32)(w - 1) * +- (OPJ_UINT32)subsampling_dx + 1; +- image->y1 = (OPJ_UINT32)parameters->image_offset_y0 + (OPJ_UINT32)(h - 1) * +- (OPJ_UINT32)subsampling_dy + 1; +- +- if (raw_cp->rawBitDepth <= 8) { +- unsigned char value = 0; +- for (compno = 0; compno < numcomps; compno++) { +- int nloop = (w * h) / (raw_cp->rawComps[compno].dx * +- raw_cp->rawComps[compno].dy); +- for (i = 0; i < nloop; i++) { +- if (!fread(&value, 1, 1, f)) { +- fprintf(stderr, "Error reading raw file. End of file probably reached.\n"); +- opj_image_destroy(image); +- fclose(f); +- return NULL; +- } +- image->comps[compno].data[i] = raw_cp->rawSigned ? (char)value : value; +- } +- } +- } else if (raw_cp->rawBitDepth <= 16) { +- unsigned short value; +- for (compno = 0; compno < numcomps; compno++) { +- int nloop = (w * h) / (raw_cp->rawComps[compno].dx * +- raw_cp->rawComps[compno].dy); +- for (i = 0; i < nloop; i++) { +- unsigned char temp1; +- unsigned char temp2; +- if (!fread(&temp1, 1, 1, f)) { +- fprintf(stderr, "Error reading raw file. End of file probably reached.\n"); +- opj_image_destroy(image); +- fclose(f); +- return NULL; +- } +- if (!fread(&temp2, 1, 1, f)) { +- fprintf(stderr, "Error reading raw file. End of file probably reached.\n"); +- opj_image_destroy(image); +- fclose(f); +- return NULL; +- } +- if (big_endian) { +- value = (unsigned short)((temp1 << 8) + temp2); +- } else { +- value = (unsigned short)((temp2 << 8) + temp1); +- } +- image->comps[compno].data[i] = raw_cp->rawSigned ? (short)value : value; +- } +- } +- } else { +- fprintf(stderr, +- "OpenJPEG cannot encode raw components with bit depth higher than 16 bits.\n"); +- opj_image_destroy(image); +- fclose(f); +- return NULL; ++ sprintf(destname, "%s", outfile); + } + +- if (fread(&ch, 1, 1, f)) { +- fprintf(stderr, "Warning. End of raw file not reached... processing anyway\n"); ++ fdest = fopen(destname, "wb"); ++ if (!fdest) { ++ fprintf(stderr, "ERROR -> failed to open %s for writing\n", destname); ++ free(destname); ++ return 1; + } +- fclose(f); ++ wr = (int)image->comps[compno].w; ++ hr = (int)image->comps[compno].h; ++ prec = (int)image->comps[compno].prec; ++ max = (1 << prec) - 1; + +- return image; +-} ++ fprintf(fdest, "P5\n#OpenJPEG-%s\n%d %d\n%d\n", opj_version(), wr, hr, max); + +-opj_image_t* rawltoimage(const char *filename, opj_cparameters_t *parameters, +- raw_cparameters_t *raw_cp) +-{ +- return rawtoimage_common(filename, parameters, raw_cp, OPJ_FALSE); +-} +- +-opj_image_t* rawtoimage(const char *filename, opj_cparameters_t *parameters, +- raw_cparameters_t *raw_cp) +-{ +- return rawtoimage_common(filename, parameters, raw_cp, OPJ_TRUE); +-} ++ red = image->comps[compno].data; + +-static int imagetoraw_common(opj_image_t * image, const char *outfile, +- OPJ_BOOL big_endian) +-{ +- FILE *rawFile = NULL; +- size_t res; +- unsigned int compno, numcomps; +- int w, h, fails; +- int line, row, curr, mask; +- int *ptr; +- unsigned char uc; +- (void)big_endian; +- +- if ((image->numcomps * image->x1 * image->y1) == 0) { +- fprintf(stderr, "\nError: invalid raw image parameters\n"); +- return 1; ++ if (!red) { ++ fclose(fdest); ++ continue; + } + +- numcomps = image->numcomps; ++ adjustR = ++ (image->comps[compno].sgnd ? 1 << (image->comps[compno].prec - 1) : 0); + +- if (numcomps > 4) { +- numcomps = 4; +- } +- +- for (compno = 1; compno < numcomps; ++compno) { +- if (image->comps[0].dx != image->comps[compno].dx) { +- break; +- } +- if (image->comps[0].dy != image->comps[compno].dy) { +- break; ++ if (prec > 8) { ++ for (i = 0; i < wr * hr; i++) { ++ v = *red + adjustR; ++ ++red; ++ if (v > 65535) { ++ v = 65535; ++ } else if (v < 0) { ++ v = 0; + } +- if (image->comps[0].prec != image->comps[compno].prec) { +- break; +- } +- if (image->comps[0].sgnd != image->comps[compno].sgnd) { +- break; +- } +- } +- if (compno != numcomps) { +- fprintf(stderr, +- "imagetoraw_common: All components shall have the same subsampling, same bit depth, same sign.\n"); +- fprintf(stderr, "\tAborting\n"); +- return 1; +- } + +- rawFile = fopen(outfile, "wb"); +- if (!rawFile) { +- fprintf(stderr, "Failed to open %s for writing !!\n", outfile); +- return 1; +- } +- +- fails = 1; +- fprintf(stdout, "Raw image characteristics: %d components\n", image->numcomps); ++ /* netpbm: */ ++ fprintf(fdest, "%c%c", (unsigned char)(v >> 8), (unsigned char)v); + +- for (compno = 0; compno < image->numcomps; compno++) { +- fprintf(stdout, "Component %u characteristics: %dx%dx%d %s\n", compno, +- image->comps[compno].w, +- image->comps[compno].h, image->comps[compno].prec, +- image->comps[compno].sgnd == 1 ? "signed" : "unsigned"); +- +- w = (int)image->comps[compno].w; +- h = (int)image->comps[compno].h; +- +- if (image->comps[compno].prec <= 8) { +- if (image->comps[compno].sgnd == 1) { +- mask = (1 << image->comps[compno].prec) - 1; +- ptr = image->comps[compno].data; +- for (line = 0; line < h; line++) { +- for (row = 0; row < w; row++) { +- curr = *ptr; +- if (curr > 127) { +- curr = 127; +- } else if (curr < -128) { +- curr = -128; +- } +- uc = (unsigned char)(curr & mask); +- res = fwrite(&uc, 1, 1, rawFile); +- if (res < 1) { +- fprintf(stderr, "failed to write 1 byte for %s\n", outfile); +- goto fin; +- } +- ptr++; +- } +- } +- } else if (image->comps[compno].sgnd == 0) { +- mask = (1 << image->comps[compno].prec) - 1; +- ptr = image->comps[compno].data; +- for (line = 0; line < h; line++) { +- for (row = 0; row < w; row++) { +- curr = *ptr; +- if (curr > 255) { +- curr = 255; +- } else if (curr < 0) { +- curr = 0; +- } +- uc = (unsigned char)(curr & mask); +- res = fwrite(&uc, 1, 1, rawFile); +- if (res < 1) { +- fprintf(stderr, "failed to write 1 byte for %s\n", outfile); +- goto fin; +- } +- ptr++; +- } +- } +- } +- } else if (image->comps[compno].prec <= 16) { +- if (image->comps[compno].sgnd == 1) { +- union { +- signed short val; +- signed char vals[2]; +- } uc16; +- mask = (1 << image->comps[compno].prec) - 1; +- ptr = image->comps[compno].data; +- for (line = 0; line < h; line++) { +- for (row = 0; row < w; row++) { +- curr = *ptr; +- if (curr > 32767) { +- curr = 32767; +- } else if (curr < -32768) { +- curr = -32768; +- } +- uc16.val = (signed short)(curr & mask); +- res = fwrite(uc16.vals, 1, 2, rawFile); +- if (res < 2) { +- fprintf(stderr, "failed to write 2 byte for %s\n", outfile); +- goto fin; +- } +- ptr++; +- } +- } +- } else if (image->comps[compno].sgnd == 0) { +- union { +- unsigned short val; +- unsigned char vals[2]; +- } uc16; +- mask = (1 << image->comps[compno].prec) - 1; +- ptr = image->comps[compno].data; +- for (line = 0; line < h; line++) { +- for (row = 0; row < w; row++) { +- curr = *ptr; +- if (curr > 65535) { +- curr = 65535; +- } else if (curr < 0) { +- curr = 0; +- } +- uc16.val = (unsigned short)(curr & mask); +- res = fwrite(uc16.vals, 1, 2, rawFile); +- if (res < 2) { +- fprintf(stderr, "failed to write 2 byte for %s\n", outfile); +- goto fin; +- } +- ptr++; +- } +- } +- } +- } else if (image->comps[compno].prec <= 32) { +- fprintf(stderr, "More than 16 bits per component not handled yet\n"); +- goto fin; +- } else { +- fprintf(stderr, "Error: invalid precision: %d\n", image->comps[compno].prec); +- goto fin; ++ if (has_alpha) { ++ v = *alpha++; ++ if (v > 65535) { ++ v = 65535; ++ } else if (v < 0) { ++ v = 0; ++ } ++ ++ /* netpbm: */ ++ fprintf(fdest, "%c%c", (unsigned char)(v >> 8), (unsigned char)v); ++ } ++ } /* for(i */ ++ } else { /* prec <= 8 */ ++ for (i = 0; i < wr * hr; ++i) { ++ v = *red + adjustR; ++ ++red; ++ if (v > 255) { ++ v = 255; ++ } else if (v < 0) { ++ v = 0; + } ++ fprintf(fdest, "%c", (unsigned char)v); ++ } + } +- fails = 0; +-fin: +- fclose(rawFile); +- return fails; +-} +- +-int imagetoraw(opj_image_t * image, const char *outfile) +-{ +- return imagetoraw_common(image, outfile, OPJ_TRUE); +-} +- +-int imagetorawl(opj_image_t * image, const char *outfile) +-{ +- return imagetoraw_common(image, outfile, OPJ_FALSE); +-} ++ fclose(fdest); ++ } /* for (compno */ ++ free(destname); + ++ return 0; ++} /* imagetopnm() */ diff --git a/oss-internship-2020/openjpeg/examples/convert_h.patch b/oss-internship-2020/openjpeg/examples/convert_h.patch new file mode 100644 index 0000000..53f729c --- /dev/null +++ b/oss-internship-2020/openjpeg/examples/convert_h.patch @@ -0,0 +1,135 @@ +--- convert.h 2020-08-27 15:46:45.028628305 +0000 ++++ convert_helper.h 2020-08-27 14:26:02.155455250 +0000 +@@ -1,126 +1,8 @@ +-/* +- * The copyright in this software is being made available under the 2-clauses +- * BSD License, included below. This software may be subject to other third +- * party and contributor rights, including patent rights, and no such rights +- * are granted under this license. +- * +- * Copyright (c) 2002-2014, Universite catholique de Louvain (UCL), Belgium +- * Copyright (c) 2002-2014, Professor Benoit Macq +- * Copyright (c) 2001-2003, David Janssens +- * Copyright (c) 2002-2003, Yannick Verschueren +- * Copyright (c) 2003-2007, Francois-Olivier Devaux +- * Copyright (c) 2003-2014, Antonin Descampe +- * Copyright (c) 2005, Herve Drolon, FreeImage Team +- * All rights reserved. +- * +- * Redistribution and use in source and binary forms, with or without +- * modification, are permitted provided that the following conditions +- * are met: +- * 1. Redistributions of source code must retain the above copyright +- * notice, this list of conditions and the following disclaimer. +- * 2. Redistributions in binary form must reproduce the above copyright +- * notice, this list of conditions and the following disclaimer in the +- * documentation and/or other materials provided with the distribution. +- * +- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS `AS IS' +- * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +- * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +- * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +- * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +- * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +- * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +- * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +- * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +- * POSSIBILITY OF SUCH DAMAGE. +- */ +-#ifndef __J2K_CONVERT_H +-#define __J2K_CONVERT_H ++// imagetopnm and the two functions it calls internaly are patched ++// versions of the library's tools; from openjpeg/src/bin/jp2/convert.c + +-/**@name RAW component encoding parameters */ +-/*@{*/ +-typedef struct raw_comp_cparameters { +- /** subsampling in X direction */ +- int dx; +- /** subsampling in Y direction */ +- int dy; +- /*@}*/ +-} raw_comp_cparameters_t; +- +-/**@name RAW image encoding parameters */ +-/*@{*/ +-typedef struct raw_cparameters { +- /** width of the raw image */ +- int rawWidth; +- /** height of the raw image */ +- int rawHeight; +- /** number of components of the raw image */ +- int rawComp; +- /** bit depth of the raw image */ +- int rawBitDepth; +- /** signed/unsigned raw image */ +- OPJ_BOOL rawSigned; +- /** raw components parameters */ +- raw_comp_cparameters_t *rawComps; +- /*@}*/ +-} raw_cparameters_t; +- +-/* Component precision clipping */ +-void clip_component(opj_image_comp_t* component, OPJ_UINT32 precision); +-/* Component precision scaling */ +-void scale_component(opj_image_comp_t* component, OPJ_UINT32 precision); +- +-/* planar / interleaved conversions */ +-typedef void (* convert_32s_CXPX)(const OPJ_INT32* pSrc, OPJ_INT32* const* pDst, +- OPJ_SIZE_T length); +-extern const convert_32s_CXPX convert_32s_CXPX_LUT[5]; +-typedef void (* convert_32s_PXCX)(OPJ_INT32 const* const* pSrc, OPJ_INT32* pDst, +- OPJ_SIZE_T length, OPJ_INT32 adjust); +-extern const convert_32s_PXCX convert_32s_PXCX_LUT[5]; +-/* bit depth conversions */ +-typedef void (* convert_XXx32s_C1R)(const OPJ_BYTE* pSrc, OPJ_INT32* pDst, +- OPJ_SIZE_T length); +-extern const convert_XXx32s_C1R convert_XXu32s_C1R_LUT[9]; /* up to 8bpp */ +-typedef void (* convert_32sXXx_C1R)(const OPJ_INT32* pSrc, OPJ_BYTE* pDst, +- OPJ_SIZE_T length); +-extern const convert_32sXXx_C1R convert_32sXXu_C1R_LUT[9]; /* up to 8bpp */ +- +- +-/* TGA conversion */ +-opj_image_t* tgatoimage(const char *filename, opj_cparameters_t *parameters); +-int imagetotga(opj_image_t * image, const char *outfile); +- +-/* BMP conversion */ +-opj_image_t* bmptoimage(const char *filename, opj_cparameters_t *parameters); +-int imagetobmp(opj_image_t *image, const char *outfile); +- +-/* TIFF conversion*/ +-opj_image_t* tiftoimage(const char *filename, opj_cparameters_t *parameters); +-int imagetotif(opj_image_t *image, const char *outfile); +-/** +-Load a single image component encoded in PGX file format +-@param filename Name of the PGX file to load +-@param parameters *List ?* +-@return Returns a greyscale image if successful, returns NULL otherwise +-*/ +-opj_image_t* pgxtoimage(const char *filename, opj_cparameters_t *parameters); +-int imagetopgx(opj_image_t *image, const char *outfile); +- +-opj_image_t* pnmtoimage(const char *filename, opj_cparameters_t *parameters); +-int imagetopnm(opj_image_t *image, const char *outfile, int force_split); +- +-/* RAW conversion */ +-int imagetoraw(opj_image_t * image, const char *outfile); +-int imagetorawl(opj_image_t * image, const char *outfile); +-opj_image_t* rawtoimage(const char *filename, opj_cparameters_t *parameters, +- raw_cparameters_t *raw_cp); +-opj_image_t* rawltoimage(const char *filename, opj_cparameters_t *parameters, +- raw_cparameters_t *raw_cp); +- +-/* PNG conversion*/ +-extern int imagetopng(opj_image_t *image, const char *write_idf); +-extern opj_image_t* pngtoimage(const char *filename, +- opj_cparameters_t *parameters); +- +-#endif /* __J2K_CONVERT_H */ ++#include "openjp2_sapi.sapi.h" + ++const char* opj_version(void); ++static int are_comps_similar(opj_image_t* image); ++int imagetopnm(opj_image_t* image, const char* outfile, int force_split); diff --git a/oss-internship-2020/openjpeg/examples/decompress_example.cc b/oss-internship-2020/openjpeg/examples/decompress_example.cc new file mode 100644 index 0000000..4d8630a --- /dev/null +++ b/oss-internship-2020/openjpeg/examples/decompress_example.cc @@ -0,0 +1,157 @@ +// Copyright 2020 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Perform decompression from *.jp2 to *.pnm format + +#include +#include + +#include +#include +#include + +#include "gen_files/convert.h" // NOLINT(build/include) +#include "openjp2_sapi.sapi.h" // NOLINT(build/include) + +class Openjp2SapiSandbox : public Openjp2Sandbox { + public: + explicit Openjp2SapiSandbox(std::string in_file) + : in_file_(std::move(in_file)) {} + + std::unique_ptr ModifyPolicy( + sandbox2::PolicyBuilder*) override { + return sandbox2::PolicyBuilder() + .AllowStaticStartup() + .AllowOpen() + .AllowRead() + .AllowWrite() + .AllowStat() + .AllowSystemMalloc() + .AllowExit() + .AllowSyscalls({ + __NR_futex, + __NR_close, + __NR_lseek, + }) + .AddFile(in_file_) + .BuildOrDie(); + } + + private: + std::string in_file_; +}; + +int main(int argc, char* argv[]) { + gflags::ParseCommandLineFlags(&argc, &argv, true); + google::InitGoogleLogging(argv[0]); + + if (argc != 3) { + std::cerr << "Usage: " << basename(argv[0]) << " absolute/path/to/INPUT.jp2" + << " absolute/path/to/OUTPUT.pnm\n"; + return EXIT_FAILURE; + } + + std::string in_file(argv[1]); + + // Initialize sandbox. + Openjp2SapiSandbox sandbox(in_file); + absl::Status status = sandbox.Init(); + CHECK(status.ok()) << "Sandbox initialization failed " << status; + + Openjp2Api api(&sandbox); + sapi::v::ConstCStr in_file_v(in_file.c_str()); + + // Initialize library's main data-holders. + sapi::StatusOr stream = + api.opj_stream_create_default_file_stream(in_file_v.PtrBefore(), 1); + CHECK(stream.ok()) << "Stream initialization failed: " << stream.status(); + sapi::v::RemotePtr stream_pointer(stream.value()); + + sapi::StatusOr codec = api.opj_create_decompress(OPJ_CODEC_JP2); + CHECK(codec.ok()) << "Codec initialization failed: " << stream.status(); + sapi::v::RemotePtr codec_pointer(codec.value()); + + sapi::v::Struct parameters; + status = api.opj_set_default_decoder_parameters(parameters.PtrBoth()); + CHECK(status.ok()) << "Parameters initialization failed " << status; + + sapi::StatusOr bool_status = + api.opj_setup_decoder(&codec_pointer, parameters.PtrBefore()); + CHECK(bool_status.ok() && bool_status.value()) << "Decoder setup failed"; + + // Start reading image from the input file. + sapi::v::GenericPtr image_pointer; + bool_status = api.opj_read_header(&stream_pointer, &codec_pointer, + image_pointer.PtrAfter()); + CHECK(bool_status.ok() && bool_status.value()) + << "Reading image header failed"; + + sapi::v::Struct image; + image.SetRemote(reinterpret_cast(image_pointer.GetValue())); + CHECK(sandbox.TransferFromSandboxee(&image).ok()) + << "Transfer from sandboxee failed"; + + bool_status = + api.opj_decode(&codec_pointer, &stream_pointer, image.PtrAfter()); + CHECK(bool_status.ok() && bool_status.value()) << "Decoding failed"; + + bool_status = api.opj_end_decompress(&codec_pointer, &stream_pointer); + CHECK(bool_status.ok() && bool_status.value()) << "Ending decompress failed"; + + int components = image.data().numcomps; + + // Transfer the read data to the main process. + sapi::v::Array image_components(components); + image_components.SetRemote(image.data().comps); + CHECK(sandbox.TransferFromSandboxee(&image_components).ok()) + << "Transfer from sandboxee failed"; + + image.mutable_data()->comps = + static_cast(image_components.GetLocal()); + + unsigned int width = static_cast(image.data().comps[0].w); + unsigned int height = static_cast(image.data().comps[0].h); + + std::vector> data(components); + sapi::v::Array image_components_data(width * height); + + for (int i = 0; i < components; ++i) { + image_components_data.SetRemote(image.data().comps[i].data); + CHECK(sandbox.TransferFromSandboxee(&image_components_data).ok()) + << "Transfer from sandboxee failed"; + + std::vector component_data( + image_components_data.GetData(), + image_components_data.GetData() + (width * height)); + data[i] = std::move(component_data); + image_components[i].data = &data[i][0]; + } + + // Convert the image to the desired format and save it to the file. + int error = + imagetopnm(static_cast(image.GetLocal()), argv[2], 0); + CHECK(!error) << "Image convert failed"; + + // Clean up. + status = api.opj_image_destroy(image.PtrNone()); + CHECK(status.ok()) << "Image destroy failed " << status; + + status = api.opj_stream_destroy(&stream_pointer); + CHECK(status.ok()) << "Stream destroy failed " << status; + + status = api.opj_destroy_codec(&codec_pointer); + CHECK(status.ok()) << "Codec destroy failed " << status; + + return EXIT_SUCCESS; +} diff --git a/oss-internship-2020/pffft/.gitignore b/oss-internship-2020/pffft/.gitignore new file mode 100644 index 0000000..eb6d948 --- /dev/null +++ b/oss-internship-2020/pffft/.gitignore @@ -0,0 +1,3 @@ +*.o +*.a +pffft_main diff --git a/oss-internship-2020/pffft/CMakeLists.txt b/oss-internship-2020/pffft/CMakeLists.txt new file mode 100644 index 0000000..af3bab4 --- /dev/null +++ b/oss-internship-2020/pffft/CMakeLists.txt @@ -0,0 +1,104 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +cmake_minimum_required(VERSION 3.10) + +project(pffft CXX C) + +set(CMAKE_CXX_STANDARD 17) +set(CMAKE_CXX_STANDARD_REQUIRED True) + +add_library(pffft STATIC + master/pffft.c + master/pffft.h + master/fftpack.c + master/fftpack.h +) + +add_executable(pffft_main + master/test_pffft.c +) + +target_link_libraries(pffft_main PRIVATE + pffft +) + +set(MATH_LIBS "") +include(CheckLibraryExists) +check_library_exists(m sin "" LIBM) +if(LIBM) + list(APPEND MATH_LIBS "m") +endif() + +target_link_libraries(pffft PUBLIC ${MATH_LIBS}) + +# Adding dependencies +set(SAPI_ROOT "../.." CACHE PATH "Path to the Sandboxed API source tree") +# Then configure: +# mkdir -p build && cd build +# cmake .. -G Ninja -DSAPI_ROOT=$HOME/sapi_root + +set(SAPI_ENABLE_EXAMPLES OFF CACHE BOOL "") +set(SAPI_ENABLE_TESTS OFF CACHE BOOL "") +add_subdirectory("${SAPI_ROOT}" + "${CMAKE_BINARY_DIR}/sandboxed-api-build" + # Omit this to have the full Sandboxed API in IDE + EXCLUDE_FROM_ALL) + +add_sapi_library(pffft_sapi + FUNCTIONS pffft_new_setup + pffft_destroy_setup + pffft_transform + pffft_transform_ordered + pffft_zreorder + pffft_zconvolve_accumulate + pffft_aligned_malloc + pffft_aligned_free + pffft_simd_size + cffti + cfftf + cfftb + rffti + rfftf + rfftb + cosqi + cosqf + cosqb + costi + cost + sinqi + sinqb + sinqf + sinti + sint + + INPUTS master/pffft.h master/fftpack.h + LIBRARY pffft + LIBRARY_NAME Pffft + + NAMESPACE "" +) + +target_include_directories(pffft_sapi INTERFACE + "${PROJECT_BINARY_DIR}" +) + +add_executable(pffft_sandboxed + main_pffft_sandboxed.cc +) + +target_link_libraries(pffft_sandboxed PRIVATE + pffft_sapi + sapi::sapi +) diff --git a/oss-internship-2020/pffft/README.md b/oss-internship-2020/pffft/README.md new file mode 100644 index 0000000..18834fc --- /dev/null +++ b/oss-internship-2020/pffft/README.md @@ -0,0 +1,87 @@ +# Sandboxing PFFFT library + +Build System: CMake +OS: Linux + +### Check out the PFFFT library & CMake set up +``` +git submodule update --init --recursive + +mkdir -p build && cd build +cmake .. -G Ninja -DPFFFT_ROOT_DIR=$PWD +ninjas +``` +### For testing: +`cd build`, then `./pffft_sandboxed` + +### For debug: +display custom info with +`./pffft_sandboxed --logtostderr` + +## ***About the project*** +*PFFFT library is concerned with 1D Fast-Fourier Transformations finding a +compromise between accuracy and speed. It deals with real and complex +vectors, both cases being illustrated in the testing part (`test_pffft.c` +for initially and original version, `main_pffft_sandboxed.cc` for our +currently implemented sandboxed version). +The original files can be found at: https://bitbucket.org/jpommier/pffft/src.* + +*The purpose of sandboxing is to limit the permissions and capabilities of +library’s methods, in order to secure the usage of them. +After obtaining the sandbox, the functions will be called through an +Sandbox API (being called `api` in the current test) and so, the +operations, system calls or namspaces access may be controlled. +From both `pffft.h` and `fftpack.h` headers, useful methods are added to +sapi library builded with CMake. There is also a need to link math library +as the transformations made require mathematical operators. +Regarding the testing of the methods, one main is doing this job by +iterating through a set of values, that represents the accuracy of +transformations and print the speed for each value and type of +transformation. More specifically, the input length is the target for +accuracy (named as `n`) and it stands for the number of data points from +the series that calculate the result of transformation. It is also +important to mention that the `complex` variable stands for a boolean value +that tells the type of transformation (0 for REAL and 1 for COMPLEX) and +it is taken into account while testing. +In the end, the performance of PFFFT library it is outlined by the output. +There are two output formats available, from which you can choose through +`--output_format=` command-line flag. +Without using this type of argument when running, the output format is set +by default.* + +#### CMake observations resume: +* linking pffft and fftpack (which contains necessary functions for pffft) +* set math library + +#### Sandboxed main observations resume: +* containing two testing parts (fft / pffft benchmarks) +* showing the performance of the transformations implies + testing them through various FFT dimenstions. + Variable n, the input length, will take specific values + meaning the number of points to which it is set the calculus + (more details of mathematical purpose of n - https://en.wikipedia.org/wiki/Cooley%E2%80%93Tukey_FFT_algorithm). +* output shows speed depending on the input length +* use `--output_format=0` or `--output_format=1` arguments to choose between output formats. + `0` is for a detailed output, while `1` is only displaying each transformation process speed. + +### Bugs history +1. [Solved] pffft benchmark bug: "Sandbox not active" + + n = 64, status OK, `pffft_transform` generates error + n > 64, status not OK + Problem on initialising `sapi::StatusOr s;` the memory that stays + for s is not the same with the address passed in `pffft_transform` function. + (`sapi::v::GenericPtr` - to be changed) + + Temporary solution: change the generated files to accept + `uintptr_t` instead of `PFFFT_Setup` + + Solution: using `sapi::v::RemotePtr` instead of `sapi::v::GenericPtr` + to access the memory of object `s` + +2. [Unresolved] compiling bug: "No space left on device" + + The building process creates some `embed` files that use lots of + memory, trying to write them on `/tmp`. + + Temporary solution: clean /tmp directory by `sudo rm -rf /tmp/*` diff --git a/oss-internship-2020/pffft/main_pffft_sandboxed.cc b/oss-internship-2020/pffft/main_pffft_sandboxed.cc new file mode 100644 index 0000000..78b4b97 --- /dev/null +++ b/oss-internship-2020/pffft/main_pffft_sandboxed.cc @@ -0,0 +1,207 @@ +// Copyright 2020 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include + +#include +#include +#include +#include +#include + +#include +#include "pffft_sapi.sapi.h" // NOLINT(build/include) +#include "sandboxed_api/util/flag.h" +#include "sandboxed_api/vars.h" + +ABSL_DECLARE_FLAG(string, sandbox2_danger_danger_permit_all); +ABSL_DECLARE_FLAG(string, sandbox2_danger_danger_permit_all_and_log); + +class PffftSapiSandbox : public PffftSandbox { + public: + std::unique_ptr ModifyPolicy(sandbox2::PolicyBuilder*) { + return sandbox2::PolicyBuilder() + .AllowStaticStartup() + .AllowOpen() + .AllowRead() + .AllowWrite() + .AllowSystemMalloc() + .AllowExit() + .AllowSyscalls({ + __NR_futex, + __NR_close, + __NR_getrusage, + }) + .BuildOrDie(); + } +}; + +// output_format flag determines whether the output shows information in detail +// or not. By default, the flag is set as 0, meaning an elaborate display +// (see ShowOutput method). +static bool ValidateFlag(const char* flagname, int32_t value) { + if (value >= 0 && value < 32768) { + return true; + } + + LOG(ERROR) << "Invalid value for --" << flagname << "."; + return false; +} + +DEFINE_int32(output_format, 0, "Value to specific the output format."); +DEFINE_validator(output_format, &ValidateFlag); + +double UclockSec() { return static_cast(clock()) / CLOCKS_PER_SEC; } + +void ShowOutput(const char* name, int n, int complex, float flops, float t0, + float t1, int max_iter) { + float mflops = flops / 1e6 / (t1 - t0 + 1e-16); + if (FLAGS_output_format) { + if (flops != -1) { + printf("|%9.0f ", mflops); + } else { + printf("| n/a "); + } + } else if (flops != -1) { + printf("n=%5d, %s %16s : %6.0f MFlops [t=%6.0f ns, %d runs]\n", n, + (complex ? "CPLX" : "REAL"), name, mflops, + (t1 - t0) / 2 / max_iter * 1e9, max_iter); + } + fflush(stdout); +} + +absl::Status PffftMain() { + LOG(INFO) << "Initializing sandbox...\n"; + + PffftSapiSandbox sandbox; + SAPI_RETURN_IF_ERROR(sandbox.Init()); + + PffftApi api(&sandbox); + + // kTransformSizes is a vector keeping the values by which iterates n, its + // value representing the input length. More concrete, n is the number of data + // points the caclulus is up to (determinating its accuracy). To show the + // performance of Fast-Fourier Transformations the program is testing for + // various values of n. + constexpr int kTransformSizes[] = { + 64, 96, 128, 160, 192, 256, 384, 5 * 96, 512, 5 * 128, + 3 * 256, 800, 1024, 2048, 2400, 4096, 8192, 9 * 1024, 16384, 32768}; + + for (int complex : {0, 1}) { + for (int n : kTransformSizes) { + const int n_float = n * (complex ? 2 : 1); + int n_bytes = n_float * sizeof(float); + + std::vector work(2 * n_float + 15, 0.0); + sapi::v::Array work_array(&work[0], work.size()); + + std::vector x(n_bytes, 0.0); + sapi::v::Array x_array(&x[0], x.size()); + + std::vector y(n_bytes, 0.0); + sapi::v::Array y_array(&y[0], y.size()); + + std::vector z(n_bytes, 0.0); + sapi::v::Array z_array(&z[0], z.size()); + + double t0; + double t1; + double flops; + + int max_iter = 5120000 / n * 4; + + for (int k = 0; k < n_float; ++k) { + x[k] = 0; + } + + // FFTPack benchmark + { + // SIMD_SZ == 4 (returning value of pffft_simd_size()) + int simd_size_iter = max_iter / 4; + + if (simd_size_iter == 0) simd_size_iter = 1; + if (complex) { + SAPI_RETURN_IF_ERROR(api.cffti(n, work_array.PtrBoth())) + } else { + SAPI_RETURN_IF_ERROR(api.rffti(n, work_array.PtrBoth())); + } + t0 = UclockSec(); + + for (int iter = 0; iter < simd_size_iter; ++iter) { + if (complex) { + SAPI_RETURN_IF_ERROR( + api.cfftf(n, x_array.PtrBoth(), work_array.PtrBoth())); + SAPI_RETURN_IF_ERROR( + api.cfftb(n, x_array.PtrBoth(), work_array.PtrBoth())); + } else { + SAPI_RETURN_IF_ERROR( + api.rfftf(n, x_array.PtrBoth(), work_array.PtrBoth())); + SAPI_RETURN_IF_ERROR( + api.rfftb(n, x_array.PtrBoth(), work_array.PtrBoth())); + } + } + t1 = UclockSec(); + + flops = (simd_size_iter * 2) * + ((complex ? 5 : 2.5) * static_cast(n) * + log(static_cast(n)) / M_LN2); + ShowOutput("FFTPack", n, complex, flops, t0, t1, simd_size_iter); + } + + // PFFFT benchmark + { + SAPI_ASSIGN_OR_RETURN( + PFFFT_Setup * s, + api.pffft_new_setup(n, complex ? PFFFT_COMPLEX : PFFFT_REAL)); + + sapi::v::RemotePtr s_reg(s); + + t0 = UclockSec(); + for (int iter = 0; iter < max_iter; ++iter) { + SAPI_RETURN_IF_ERROR( + api.pffft_transform(&s_reg, x_array.PtrBoth(), z_array.PtrBoth(), + y_array.PtrBoth(), PFFFT_FORWARD)); + SAPI_RETURN_IF_ERROR( + api.pffft_transform(&s_reg, x_array.PtrBoth(), z_array.PtrBoth(), + y_array.PtrBoth(), PFFFT_FORWARD)); + } + + t1 = UclockSec(); + SAPI_RETURN_IF_ERROR(api.pffft_destroy_setup(&s_reg)); + + flops = (max_iter * 2) * ((complex ? 5 : 2.5) * static_cast(n) * + log(static_cast(n)) / M_LN2); + ShowOutput("PFFFT", n, complex, flops, t0, t1, max_iter); + + LOG(INFO) << "n = " << n << " SUCCESSFULLY"; + } + } + } + + return absl::OkStatus(); +} + +int main(int argc, char* argv[]) { + // Initialize Google's logging library. + google::InitGoogleLogging(argv[0]); + + gflags::ParseCommandLineFlags(&argc, &argv, true); + + if (absl::Status status = PffftMain(); !status.ok()) { + LOG(ERROR) << "Initialization failed: " << status.ToString(); + return EXIT_FAILURE; + } + + return EXIT_SUCCESS; +} diff --git a/sandboxed_api/BUILD.bazel b/sandboxed_api/BUILD.bazel index fbe40b3..59ed83a 100644 --- a/sandboxed_api/BUILD.bazel +++ b/sandboxed_api/BUILD.bazel @@ -147,10 +147,10 @@ cc_library( ":var_type", "//sandboxed_api/sandbox2:comms", "//sandboxed_api/util:status", - "//sandboxed_api/util:statusor", "@com_google_absl//absl/base:core_headers", "@com_google_absl//absl/memory", "@com_google_absl//absl/status", + "@com_google_absl//absl/status:statusor", "@com_google_absl//absl/strings", "@com_google_absl//absl/strings:str_format", "@com_google_absl//absl/synchronization", diff --git a/sandboxed_api/CMakeLists.txt b/sandboxed_api/CMakeLists.txt index 0f4ca85..c2b6d2f 100644 --- a/sandboxed_api/CMakeLists.txt +++ b/sandboxed_api/CMakeLists.txt @@ -43,6 +43,8 @@ add_library(sapi_embed_file STATIC add_library(sapi::embed_file ALIAS sapi_embed_file) target_link_libraries(sapi_embed_file PRIVATE absl::flat_hash_map + absl::status + absl::statusor absl::strings absl::synchronization glog::glog @@ -65,6 +67,8 @@ add_library(sapi::sapi ALIAS sapi_sapi) target_link_libraries(sapi_sapi PRIVATE absl::flat_hash_map absl::memory + absl::status + absl::statusor absl::str_format absl::strings absl::synchronization @@ -76,7 +80,6 @@ target_link_libraries(sapi_sapi sandbox2::strerror sandbox2::util sapi::embed_file - sapi::status sapi::vars PUBLIC absl::core_headers sandbox2::client @@ -137,6 +140,8 @@ add_library(sapi_vars STATIC add_library(sapi::vars ALIAS sapi_vars) target_link_libraries(sapi_vars PRIVATE absl::core_headers + absl::status + absl::statusor absl::str_format absl::strings absl::synchronization @@ -147,7 +152,6 @@ target_link_libraries(sapi_vars PRIVATE sapi::lenval_core sapi::proto_arg_proto sapi::status - sapi::statusor sapi::var_type ) @@ -178,6 +182,7 @@ if(SAPI_ENABLE_TESTS) ) target_link_libraries(sapi_test PRIVATE absl::memory + absl::status benchmark sapi::sapi sapi::status diff --git a/sandboxed_api/bazel/BUILD b/sandboxed_api/bazel/BUILD index 78acfea..2448930 100644 --- a/sandboxed_api/bazel/BUILD +++ b/sandboxed_api/bazel/BUILD @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +load("@bazel_skylib//:bzl_library.bzl", "bzl_library") + licenses(["notice"]) exports_files([ @@ -19,3 +21,33 @@ exports_files([ "embed_data.bzl", "sapi.bzl", ]) + +bzl_library( + name = "build_defs_bzl", + srcs = ["build_defs.bzl"], + visibility = ["//visibility:private"], +) + +bzl_library( + name = "embed_data_bzl", + srcs = ["embed_data.bzl"], + visibility = ["//visibility:private"], +) + +bzl_library( + name = "proto_bzl", + srcs = ["proto.bzl"], + visibility = ["//visibility:private"], +) + +bzl_library( + name = "repositories_bzl", + srcs = ["repositories.bzl"], + visibility = ["//visibility:private"], +) + +bzl_library( + name = "sapi_deps_bzl", + srcs = ["sapi_deps.bzl"], + visibility = ["//visibility:private"], +) diff --git a/sandboxed_api/bazel/embed_data.bzl b/sandboxed_api/bazel/embed_data.bzl index 85ce1ba..dc12c4f 100644 --- a/sandboxed_api/bazel/embed_data.bzl +++ b/sandboxed_api/bazel/embed_data.bzl @@ -16,8 +16,8 @@ _FILEWRAPPER = "//sandboxed_api/tools/filewrapper" -# TODO(cblichmann): Convert this is to use a "_cc_toolchain" once Bazel #4370 -# is fixed. +# TODO(cblichmann): Convert this to use a "_cc_toolchain" once Bazel #4370 is +# fixed. def _sapi_cc_embed_data_impl(ctx): cc_file_artifact = None h_file_artifact = None diff --git a/sandboxed_api/bazel/proto.bzl b/sandboxed_api/bazel/proto.bzl index 43444c8..c8970d1 100644 --- a/sandboxed_api/bazel/proto.bzl +++ b/sandboxed_api/bazel/proto.bzl @@ -56,7 +56,7 @@ def sapi_proto_library( deps = [], alwayslink = False, **kwargs): - """Generates proto targets in various languages. + """Generates proto library and C++ targets. Args: name: Name for proto_library and base for the cc_proto_library name, name + diff --git a/sandboxed_api/bazel/sapi.bzl b/sandboxed_api/bazel/sapi.bzl index 11f26cc..211d5a6 100644 --- a/sandboxed_api/bazel/sapi.bzl +++ b/sandboxed_api/bazel/sapi.bzl @@ -254,9 +254,8 @@ def sapi_library( native.genrule( name = name + ".isystem", outs = [name + ".isystem.list"], - cmd = """echo | - $(CC) -E -x c++ - -v 2>&1 | - awk '/> search starts here:/{flag=1;next}/End of search/{flag=0}flag' > $@ + cmd = """$(CC) -E -x c++ -v /dev/null 2>&1 | + awk '/> search starts here:/{f=1;next}/^End of search/{f=0}f{print $$1}' > $@ """, toolchains = ["@bazel_tools//tools/cpp:current_cc_toolchain"], ) diff --git a/sandboxed_api/bazel/sapi_deps.bzl b/sandboxed_api/bazel/sapi_deps.bzl index ade3211..ceff2f4 100644 --- a/sandboxed_api/bazel/sapi_deps.bzl +++ b/sandboxed_api/bazel/sapi_deps.bzl @@ -34,9 +34,9 @@ def sapi_deps(): maybe( http_archive, name = "com_google_absl", - sha256 = "6668ada01192e2b95b42bb3668cfa5282c047de5176f5e567028e12f8bfb8aef", # 2020-04-28 - strip_prefix = "abseil-cpp-6e18c7115df9b7ca0987cc346b1b1d4b3cc829b3", - urls = ["https://github.com/abseil/abseil-cpp/archive/6e18c7115df9b7ca0987cc346b1b1d4b3cc829b3.zip"], + sha256 = "8061df0ebbd3f599bcd3f5e57fb8003564d50a9b6a81a7f968fb0196b952365d", # 2020-09-02 + strip_prefix = "abseil-cpp-0e9921b75a0fdd639a504ec8443fc1fe801becd7", + urls = ["https://github.com/abseil/abseil-cpp/archive/0e9921b75a0fdd639a504ec8443fc1fe801becd7.zip"], ) maybe( http_archive, diff --git a/sandboxed_api/call.h b/sandboxed_api/call.h index a33306a..06def7e 100644 --- a/sandboxed_api/call.h +++ b/sandboxed_api/call.h @@ -23,8 +23,8 @@ namespace sapi { namespace comms { struct ReallocRequest { - uint64_t old_addr; - uint64_t size; + uintptr_t old_addr; + size_t size; }; // Types of TAGs used with Comms channel. diff --git a/sandboxed_api/client.cc b/sandboxed_api/client.cc index df2b24d..9d490b6 100644 --- a/sandboxed_api/client.cc +++ b/sandboxed_api/client.cc @@ -229,7 +229,7 @@ void HandleCallMsg(const FuncCall& call, FuncRet* ret) { } // Handles requests to allocate memory inside the sandboxee. -void HandleAllocMsg(const uintptr_t size, FuncRet* ret) { +void HandleAllocMsg(const size_t size, FuncRet* ret) { VLOG(1) << "HandleAllocMsg: size=" << size; ret->ret_type = v::Type::kPointer; @@ -237,16 +237,15 @@ void HandleAllocMsg(const uintptr_t size, FuncRet* ret) { // Memory is copied to the pointer using an API that the memory sanitizer // is blind to (process_vm_writev). Initialize the memory here so that // the sandboxed code can still be tested with the memory sanitizer. - ret->int_val = - reinterpret_cast(calloc(1, static_cast(size))); + ret->int_val = reinterpret_cast(calloc(1, size)); #else - ret->int_val = reinterpret_cast(malloc(static_cast(size))); + ret->int_val = reinterpret_cast(malloc(size)); #endif ret->success = true; } // Like HandleAllocMsg(), but handles requests to reallocate memory. -void HandleReallocMsg(uintptr_t ptr, uintptr_t size, FuncRet* ret) { +void HandleReallocMsg(uintptr_t ptr, size_t size, FuncRet* ret) { VLOG(1) << "HandleReallocMsg(" << absl::StrCat(absl::Hex(ptr)) << ", " << size << ")"; #ifdef MEMORY_SANITIZER @@ -254,8 +253,8 @@ void HandleReallocMsg(uintptr_t ptr, uintptr_t size, FuncRet* ret) { __sanitizer_get_allocated_size(reinterpret_cast(ptr)); #endif ret->ret_type = v::Type::kPointer; - ret->int_val = reinterpret_cast( - realloc(const_cast(reinterpret_cast(ptr)), size)); + ret->int_val = + reinterpret_cast(realloc(reinterpret_cast(ptr), size)); ret->success = true; #ifdef MEMORY_SANITIZER // Memory is copied to the pointer using an API that the memory sanitizer @@ -273,7 +272,7 @@ void HandleReallocMsg(uintptr_t ptr, uintptr_t size, FuncRet* ret) { void HandleFreeMsg(uintptr_t ptr, FuncRet* ret) { VLOG(1) << "HandleFreeMsg: free(0x" << absl::StrCat(absl::Hex(ptr)) << ")"; - free(const_cast(reinterpret_cast(ptr))); + free(reinterpret_cast(ptr)); ret->ret_type = v::Type::kVoid; ret->success = true; ret->int_val = 0ULL; @@ -363,14 +362,13 @@ void ServeRequest(sandbox2::Comms* comms) { break; case comms::kMsgAllocate: VLOG(1) << "Client::kMsgAllocate"; - HandleAllocMsg(BytesAs(bytes), &ret); + HandleAllocMsg(BytesAs(bytes), &ret); break; case comms::kMsgReallocate: VLOG(1) << "Client::kMsgReallocate"; { auto req = BytesAs(bytes); - HandleReallocMsg(static_cast(req.old_addr), - static_cast(req.size), &ret); + HandleReallocMsg(req.old_addr, req.size, &ret); } break; case comms::kMsgFree: diff --git a/sandboxed_api/examples/stringop/main_stringop.cc b/sandboxed_api/examples/stringop/main_stringop.cc index d83dbf9..c7e5b2e 100644 --- a/sandboxed_api/examples/stringop/main_stringop.cc +++ b/sandboxed_api/examples/stringop/main_stringop.cc @@ -113,7 +113,7 @@ TEST(StringopTest, RawStringReversal) { { // Let's call it again with different data as argument, reusing the // existing LenVal object. - EXPECT_THAT(param.ResizeData(sandbox.GetRpcChannel(), 16), IsOk()); + EXPECT_THAT(param.ResizeData(sandbox.rpc_channel(), 16), IsOk()); memcpy(param.GetData() + 10, "ABCDEF", 6); absl::string_view data(reinterpret_cast(param.GetData()), param.GetDataSize()); @@ -134,8 +134,8 @@ TEST(StringopTest, RawStringLength) { ASSERT_THAT(sandbox.Init(), IsOk()); StringopApi api(&sandbox); SAPI_ASSERT_OK_AND_ASSIGN(void* target_mem_ptr, api.get_raw_c_string()); - SAPI_ASSERT_OK_AND_ASSIGN(uint64_t len, - sandbox.GetRpcChannel()->Strlen(target_mem_ptr)); + SAPI_ASSERT_OK_AND_ASSIGN(size_t len, + sandbox.rpc_channel()->Strlen(target_mem_ptr)); EXPECT_THAT(len, Eq(10)); } @@ -144,8 +144,8 @@ TEST(StringopTest, RawStringReading) { ASSERT_THAT(sandbox.Init(), IsOk()); StringopApi api(&sandbox); SAPI_ASSERT_OK_AND_ASSIGN(void* target_mem_ptr, api.get_raw_c_string()); - SAPI_ASSERT_OK_AND_ASSIGN(uint64_t len, - sandbox.GetRpcChannel()->Strlen(target_mem_ptr)); + SAPI_ASSERT_OK_AND_ASSIGN(size_t len, + sandbox.rpc_channel()->Strlen(target_mem_ptr)); EXPECT_THAT(len, Eq(10)); SAPI_ASSERT_OK_AND_ASSIGN(std::string data, diff --git a/sandboxed_api/examples/sum/main_sum.cc b/sandboxed_api/examples/sum/main_sum.cc index 886be2c..51e233f 100644 --- a/sandboxed_api/examples/sum/main_sum.cc +++ b/sandboxed_api/examples/sum/main_sum.cc @@ -161,7 +161,7 @@ absl::Status SumTransaction::Main() { LOG(INFO) << "Read from /proc/self/comm = [" << buffer << "]"; // Close test. - SAPI_RETURN_IF_ERROR(fd2.CloseRemoteFd(sandbox()->GetRpcChannel())); + SAPI_RETURN_IF_ERROR(fd2.CloseRemoteFd(sandbox()->rpc_channel())); memset(buffer, 0, sizeof(buffer)); SAPI_RETURN_IF_ERROR(sandbox()->Call("read", &ret, &fd2, buf.PtrBoth(), &size)); LOG(INFO) << "Read from closed /proc/self/comm = [" << buffer << "]"; diff --git a/sandboxed_api/examples/zlib/BUILD.bazel b/sandboxed_api/examples/zlib/BUILD.bazel index d318e48..3716280 100644 --- a/sandboxed_api/examples/zlib/BUILD.bazel +++ b/sandboxed_api/examples/zlib/BUILD.bazel @@ -14,11 +14,11 @@ # Description: Sandboxed API reimplementation of zlib's zpipe.c example. -licenses(["notice"]) - load("//sandboxed_api/bazel:build_defs.bzl", "sapi_platform_copts") load("//sandboxed_api/bazel:sapi.bzl", "sapi_library") +licenses(["notice"]) + sapi_library( name = "zlib-sapi", srcs = [], @@ -43,5 +43,6 @@ cc_binary( "//sandboxed_api:vars", "//sandboxed_api/util:flags", "@com_google_absl//absl/base:core_headers", + "@com_google_absl//absl/status:statusor", ], ) diff --git a/sandboxed_api/examples/zlib/CMakeLists.txt b/sandboxed_api/examples/zlib/CMakeLists.txt index 561b6d3..5750c7d 100644 --- a/sandboxed_api/examples/zlib/CMakeLists.txt +++ b/sandboxed_api/examples/zlib/CMakeLists.txt @@ -32,11 +32,11 @@ add_executable(main_zlib main_zlib.cc ) target_link_libraries(main_zlib PRIVATE + absl::status sapi::base glog::glog sapi::flags sapi::sapi sapi::status - sapi::statusor sapi::zlib_sapi ) diff --git a/sandboxed_api/examples/zlib/main_zlib.cc b/sandboxed_api/examples/zlib/main_zlib.cc index 969ae33..e6e2b2a 100644 --- a/sandboxed_api/examples/zlib/main_zlib.cc +++ b/sandboxed_api/examples/zlib/main_zlib.cc @@ -20,6 +20,7 @@ #include #include "absl/base/macros.h" #include "sandboxed_api/util/flag.h" +#include "absl/status/statusor.h" #include "sandboxed_api/examples/zlib/zlib-sapi.sapi.h" #include "sandboxed_api/examples/zlib/zlib-sapi_embed.h" #include "sandboxed_api/vars.h" @@ -47,7 +48,7 @@ int main(int argc, char** argv) { << status.message(); } - sapi::StatusOr ret; + absl::StatusOr ret; int flush; unsigned have; sapi::v::Struct strm; diff --git a/sandboxed_api/proto_helper.h b/sandboxed_api/proto_helper.h index 53ac26d..63c4294 100644 --- a/sandboxed_api/proto_helper.h +++ b/sandboxed_api/proto_helper.h @@ -22,13 +22,13 @@ #include #include "absl/status/status.h" +#include "absl/status/statusor.h" #include "sandboxed_api/proto_arg.pb.h" -#include "sandboxed_api/util/statusor.h" namespace sapi { template -sapi::StatusOr> SerializeProto(const T& proto) { +absl::StatusOr> SerializeProto(const T& proto) { static_assert(std::is_base_of::value, "Template argument must be a proto message"); // Wrap protobuf in a envelope so that we know the name of the protobuf @@ -46,7 +46,7 @@ sapi::StatusOr> SerializeProto(const T& proto) { } template -sapi::StatusOr DeserializeProto(const char* data, size_t len) { +absl::StatusOr DeserializeProto(const char* data, size_t len) { static_assert(std::is_base_of::value, "Template argument must be a proto message"); ProtoArg envelope; diff --git a/sandboxed_api/rpcchannel.cc b/sandboxed_api/rpcchannel.cc index 72a9250..72a057e 100644 --- a/sandboxed_api/rpcchannel.cc +++ b/sandboxed_api/rpcchannel.cc @@ -15,6 +15,7 @@ #include "sandboxed_api/rpcchannel.h" #include +#include "absl/status/statusor.h" #include "absl/strings/str_cat.h" #include "absl/synchronization/mutex.h" #include "sandboxed_api/call.h" @@ -26,8 +27,7 @@ namespace sapi { absl::Status RPCChannel::Call(const FuncCall& call, uint32_t tag, FuncRet* ret, v::Type exp_type) { absl::MutexLock lock(&mutex_); - if (!comms_->SendTLV(tag, sizeof(call), - reinterpret_cast(&call))) { + if (!comms_->SendTLV(tag, sizeof(call), &call)) { return absl::UnavailableError("Sending TLV value failed"); } SAPI_ASSIGN_OR_RETURN(auto fret, Return(exp_type)); @@ -35,9 +35,9 @@ absl::Status RPCChannel::Call(const FuncCall& call, uint32_t tag, FuncRet* ret, return absl::OkStatus(); } -sapi::StatusOr RPCChannel::Return(v::Type exp_type) { +absl::StatusOr RPCChannel::Return(v::Type exp_type) { uint32_t tag; - uint64_t len; + size_t len; FuncRet ret; if (!comms_->RecvTLV(&tag, &len, &ret, sizeof(ret))) { return absl::UnavailableError("Receiving TLV value failed"); @@ -66,9 +66,7 @@ sapi::StatusOr RPCChannel::Return(v::Type exp_type) { absl::Status RPCChannel::Allocate(size_t size, void** addr) { absl::MutexLock lock(&mutex_); - uint64_t sz = size; - if (!comms_->SendTLV(comms::kMsgAllocate, sizeof(sz), - reinterpret_cast(&sz))) { + if (!comms_->SendTLV(comms::kMsgAllocate, sizeof(size), &size)) { return absl::UnavailableError("Sending TLV value failed"); } @@ -80,12 +78,13 @@ absl::Status RPCChannel::Allocate(size_t size, void** addr) { absl::Status RPCChannel::Reallocate(void* old_addr, size_t size, void** new_addr) { absl::MutexLock lock(&mutex_); - comms::ReallocRequest req; - req.old_addr = reinterpret_cast(old_addr); - req.size = size; + comms::ReallocRequest req = { + .old_addr = reinterpret_cast(old_addr), + .size = size, + }; if (!comms_->SendTLV(comms::kMsgReallocate, sizeof(comms::ReallocRequest), - reinterpret_cast(&req))) { + &req)) { return absl::UnavailableError("Sending TLV value failed"); } @@ -102,9 +101,8 @@ absl::Status RPCChannel::Reallocate(void* old_addr, size_t size, absl::Status RPCChannel::Free(void* addr) { absl::MutexLock lock(&mutex_); - uint64_t remote = reinterpret_cast(addr); - if (!comms_->SendTLV(comms::kMsgFree, sizeof(remote), - reinterpret_cast(&remote))) { + uintptr_t remote = reinterpret_cast(addr); + if (!comms_->SendTLV(comms::kMsgFree, sizeof(remote), &remote)) { return absl::UnavailableError("Sending TLV value failed"); } @@ -117,8 +115,7 @@ absl::Status RPCChannel::Free(void* addr) { absl::Status RPCChannel::Symbol(const char* symname, void** addr) { absl::MutexLock lock(&mutex_); - if (!comms_->SendTLV(comms::kMsgSymbol, strlen(symname) + 1, - reinterpret_cast(symname))) { + if (!comms_->SendTLV(comms::kMsgSymbol, strlen(symname) + 1, symname)) { return absl::UnavailableError("Sending TLV value failed"); } @@ -136,9 +133,8 @@ absl::Status RPCChannel::Exit() { // Try the RPC exit sequence. But, the only thing that matters as a success // indicator is whether the Comms channel had been closed - bool unused = true; - comms_->SendTLV(comms::kMsgExit, sizeof(unused), - reinterpret_cast(&unused)); + comms_->SendTLV(comms::kMsgExit, 0, nullptr); + bool unused; comms_->RecvBool(&unused); if (!comms_->IsTerminated()) { @@ -153,9 +149,7 @@ absl::Status RPCChannel::Exit() { absl::Status RPCChannel::SendFD(int local_fd, int* remote_fd) { absl::MutexLock lock(&mutex_); - bool unused = true; - if (!comms_->SendTLV(comms::kMsgSendFd, sizeof(unused), - reinterpret_cast(&unused))) { + if (!comms_->SendTLV(comms::kMsgSendFd, 0, nullptr)) { return absl::UnavailableError("Sending TLV value failed"); } if (!comms_->SendFD(local_fd)) { @@ -172,8 +166,7 @@ absl::Status RPCChannel::SendFD(int local_fd, int* remote_fd) { absl::Status RPCChannel::RecvFD(int remote_fd, int* local_fd) { absl::MutexLock lock(&mutex_); - if (!comms_->SendTLV(comms::kMsgRecvFd, sizeof(remote_fd), - reinterpret_cast(&remote_fd))) { + if (!comms_->SendTLV(comms::kMsgRecvFd, sizeof(remote_fd), &remote_fd)) { return absl::UnavailableError("Sending TLV value failed"); } @@ -190,8 +183,7 @@ absl::Status RPCChannel::RecvFD(int remote_fd, int* local_fd) { absl::Status RPCChannel::Close(int remote_fd) { absl::MutexLock lock(&mutex_); - if (!comms_->SendTLV(comms::kMsgClose, sizeof(remote_fd), - reinterpret_cast(&remote_fd))) { + if (!comms_->SendTLV(comms::kMsgClose, sizeof(remote_fd), &remote_fd)) { return absl::UnavailableError("Sending TLV value failed"); } @@ -202,10 +194,9 @@ absl::Status RPCChannel::Close(int remote_fd) { return absl::OkStatus(); } -sapi::StatusOr RPCChannel::Strlen(void* str) { +absl::StatusOr RPCChannel::Strlen(void* str) { absl::MutexLock lock(&mutex_); - if (!comms_->SendTLV(comms::kMsgStrlen, sizeof(str), - reinterpret_cast(&str))) { + if (!comms_->SendTLV(comms::kMsgStrlen, sizeof(str), &str)) { return absl::UnavailableError("Sending TLV value failed"); } diff --git a/sandboxed_api/rpcchannel.h b/sandboxed_api/rpcchannel.h index ed50326..56d7ed8 100644 --- a/sandboxed_api/rpcchannel.h +++ b/sandboxed_api/rpcchannel.h @@ -18,11 +18,11 @@ #include #include "absl/status/status.h" +#include "absl/status/statusor.h" #include "absl/synchronization/mutex.h" #include "sandboxed_api/call.h" #include "sandboxed_api/sandbox2/comms.h" #include "sandboxed_api/var_type.h" -#include "sandboxed_api/util/statusor.h" namespace sapi { @@ -61,13 +61,13 @@ class RPCChannel { absl::Status Close(int remote_fd); // Returns length of a null-terminated c-style string (invokes strlen). - sapi::StatusOr Strlen(void* str); + absl::StatusOr Strlen(void* str); sandbox2::Comms* comms() const { return comms_; } private: // Receives the result after a call. - sapi::StatusOr Return(v::Type exp_type); + absl::StatusOr Return(v::Type exp_type); sandbox2::Comms* comms_; // Owned by sandbox2; absl::Mutex mutex_; diff --git a/sandboxed_api/sandbox.cc b/sandboxed_api/sandbox.cc index 79d93b1..d6b1bf2 100644 --- a/sandboxed_api/sandbox.cc +++ b/sandboxed_api/sandbox.cc @@ -84,7 +84,9 @@ void InitDefaultPolicyBuilder(sandbox2::PolicyBuilder* builder) { __NR_kill, __NR_tgkill, __NR_tkill, +#ifdef __NR_readlink __NR_readlink, +#endif #ifdef __NR_arch_prctl // x86-64 only __NR_arch_prctl, #endif @@ -100,7 +102,7 @@ void InitDefaultPolicyBuilder(sandbox2::PolicyBuilder* builder) { } void Sandbox::Terminate(bool attempt_graceful_exit) { - if (!IsActive()) { + if (!is_active()) { return; } @@ -129,7 +131,7 @@ static std::string PathToSAPILib(const std::string& lib_path) { absl::Status Sandbox::Init() { // It's already initialized - if (IsActive()) { + if (is_active()) { return absl::OkStatus(); } @@ -156,7 +158,7 @@ absl::Status Sandbox::Init() { } } - std::vector args{lib_path}; + std::vector args = {lib_path}; // Additional arguments, if needed. GetArgs(&args); std::vector envs{}; @@ -214,24 +216,24 @@ absl::Status Sandbox::Init() { return absl::OkStatus(); } -bool Sandbox::IsActive() const { return s2_ && !s2_->IsTerminated(); } +bool Sandbox::is_active() const { return s2_ && !s2_->IsTerminated(); } absl::Status Sandbox::Allocate(v::Var* var, bool automatic_free) { - if (!IsActive()) { + if (!is_active()) { return absl::UnavailableError("Sandbox not active"); } return var->Allocate(GetRpcChannel(), automatic_free); } absl::Status Sandbox::Free(v::Var* var) { - if (!IsActive()) { + if (!is_active()) { return absl::UnavailableError("Sandbox not active"); } return var->Free(GetRpcChannel()); } absl::Status Sandbox::SynchronizePtrBefore(v::Callable* ptr) { - if (!IsActive()) { + if (!is_active()) { return absl::UnavailableError("Sandbox not active"); } if (ptr->GetType() != v::Type::kPointer) { @@ -259,11 +261,11 @@ absl::Status Sandbox::SynchronizePtrBefore(v::Callable* ptr) { VLOG(3) << "Synchronization (TO), ptr " << p << ", Type: " << p->GetSyncType() << " for var: " << p->GetPointedVar()->ToString(); - return p->GetPointedVar()->TransferToSandboxee(GetRpcChannel(), GetPid()); + return p->GetPointedVar()->TransferToSandboxee(GetRpcChannel(), pid()); } absl::Status Sandbox::SynchronizePtrAfter(v::Callable* ptr) const { - if (!IsActive()) { + if (!is_active()) { return absl::UnavailableError("Sandbox not active"); } if (ptr->GetType() != v::Type::kPointer) { @@ -287,12 +289,12 @@ absl::Status Sandbox::SynchronizePtrAfter(v::Callable* ptr) const { p->ToString())); } - return p->GetPointedVar()->TransferFromSandboxee(GetRpcChannel(), GetPid()); + return p->GetPointedVar()->TransferFromSandboxee(GetRpcChannel(), pid()); } absl::Status Sandbox::Call(const std::string& func, v::Callable* ret, std::initializer_list args) { - if (!IsActive()) { + if (!is_active()) { return absl::UnavailableError("Sandbox not active"); } // Send data. @@ -371,29 +373,29 @@ absl::Status Sandbox::Call(const std::string& func, v::Callable* ret, } absl::Status Sandbox::Symbol(const char* symname, void** addr) { - if (!IsActive()) { + if (!is_active()) { return absl::UnavailableError("Sandbox not active"); } return rpc_channel_->Symbol(symname, addr); } absl::Status Sandbox::TransferToSandboxee(v::Var* var) { - if (!IsActive()) { + if (!is_active()) { return absl::UnavailableError("Sandbox not active"); } - return var->TransferToSandboxee(GetRpcChannel(), GetPid()); + return var->TransferToSandboxee(GetRpcChannel(), pid()); } absl::Status Sandbox::TransferFromSandboxee(v::Var* var) { - if (!IsActive()) { + if (!is_active()) { return absl::UnavailableError("Sandbox not active"); } - return var->TransferFromSandboxee(GetRpcChannel(), GetPid()); + return var->TransferFromSandboxee(GetRpcChannel(), pid()); } -sapi::StatusOr Sandbox::GetCString(const v::RemotePtr& str, - uint64_t max_length) { - if (!IsActive()) { +absl::StatusOr Sandbox::GetCString(const v::RemotePtr& str, + size_t max_length) { + if (!is_active()) { return absl::UnavailableError("Sandbox not active"); } @@ -436,22 +438,25 @@ const sandbox2::Result& Sandbox::AwaitResult() { return result_; } -absl::Status Sandbox::SetWallTimeLimit(time_t limit) const { - if (!IsActive()) { +absl::Status Sandbox::SetWallTimeLimit(absl::Duration limit) const { + if (!is_active()) { return absl::UnavailableError("Sandbox not active"); } - s2_->SetWallTimeLimit(limit); + s2_->set_walltime_limit(limit); return absl::OkStatus(); } +absl::Status Sandbox::SetWallTimeLimit(time_t limit) const { + return SetWallTimeLimit(absl::Seconds(limit)); +} + void Sandbox::Exit() const { - if (!IsActive()) { + if (!is_active()) { return; } - // Give it 1 second - s2_->SetWallTimeLimit(1); + s2_->set_walltime_limit(absl::Seconds(1)); if (!rpc_channel_->Exit().ok()) { - LOG(WARNING) << "rpc_channel->Exit() failed, killing PID: " << GetPid(); + LOG(WARNING) << "rpc_channel->Exit() failed, killing PID: " << pid(); s2_->Kill(); } } diff --git a/sandboxed_api/sandbox.h b/sandboxed_api/sandbox.h index cb49a45..576a144 100644 --- a/sandboxed_api/sandbox.h +++ b/sandboxed_api/sandbox.h @@ -35,18 +35,21 @@ namespace sapi { // means to communicate with it (make function calls, transfer memory). class Sandbox { public: + explicit Sandbox(const FileToc* embed_lib_toc) + : embed_lib_toc_(embed_lib_toc) {} + Sandbox(const Sandbox&) = delete; Sandbox& operator=(const Sandbox&) = delete; - explicit Sandbox(const FileToc* embed_lib_toc) - : comms_(nullptr), pid_(0), embed_lib_toc_(embed_lib_toc) {} virtual ~Sandbox(); // Initializes a new sandboxing session. absl::Status Init(); - // Is the current sandboxing session alive? - bool IsActive() const; + ABSL_DEPRECATED("Use sapi::Sandbox::is_active() instead") + bool IsActive() const { return is_active(); } + // Returns whether the current sandboxing session is active. + bool is_active() const; // Terminates the current sandboxing session (if it exists). void Terminate(bool attempt_graceful_exit = true); @@ -60,9 +63,13 @@ class Sandbox { // Getters for common fields. sandbox2::Comms* comms() const { return comms_; } + ABSL_DEPRECATED("Use sapi::Sandbox::rpc_channel() instead") RPCChannel* GetRpcChannel() const { return rpc_channel_.get(); } + RPCChannel* rpc_channel() const { return rpc_channel_.get(); } + ABSL_DEPRECATED("Use sapi::Sandbox::pid() instead") int GetPid() const { return pid_; } + int pid() const { return pid_; } // Synchronizes the underlying memory for the pointer before the call. absl::Status SynchronizePtrBefore(v::Callable* ptr); @@ -87,7 +94,7 @@ class Sandbox { // Frees memory in the sandboxee. absl::Status Free(v::Var* var); - // Finds address of a symbol in the sandboxee. + // Finds the address of a symbol in the sandboxee. absl::Status Symbol(const char* symname, void** addr); // Transfers memory (both directions). Status is returned (memory transfer @@ -95,14 +102,18 @@ class Sandbox { absl::Status TransferToSandboxee(v::Var* var); absl::Status TransferFromSandboxee(v::Var* var); - sapi::StatusOr GetCString(const v::RemotePtr& str, - uint64_t max_length = 10 * 1024 * - 1024); + absl::StatusOr GetCString(const v::RemotePtr& str, + size_t max_length = 10ULL + << 20 /* 10 MiB*/ + ); // Waits until the sandbox terminated and returns the result. const sandbox2::Result& AwaitResult(); const sandbox2::Result& result() const { return result_; } + absl::Status SetWallTimeLimit(absl::Duration limit) const; + ABSL_DEPRECATED( + "Use sapi::Sandbox::SetWallTimeLimit(absl::Duration) overload instead") absl::Status SetWallTimeLimit(time_t limit) const; protected: @@ -144,11 +155,11 @@ class Sandbox { sandbox2::Result result_; // Comms with the sandboxee. - sandbox2::Comms* comms_; + sandbox2::Comms* comms_ = nullptr; // RPCChannel object. std::unique_ptr rpc_channel_; // The main pid of the sandboxee. - pid_t pid_; + pid_t pid_ = 0; // FileTOC with the embedded library, takes precedence over GetLibPath if // present (not nullptr). diff --git a/sandboxed_api/sandbox2/BUILD.bazel b/sandboxed_api/sandbox2/BUILD.bazel index 09dbab4..839a7e0 100644 --- a/sandboxed_api/sandbox2/BUILD.bazel +++ b/sandboxed_api/sandbox2/BUILD.bazel @@ -26,6 +26,13 @@ licenses(["notice"]) # Apache 2.0 exports_files(["testdata/hostname"]) +cc_library( + name = "config", + hdrs = ["config.h"], + copts = sapi_platform_copts(), + deps = ["@com_google_absl//absl/base:config"], +) + cc_library( name = "bpfdisassembler", srcs = ["bpfdisassembler.cc"], @@ -40,6 +47,7 @@ cc_library( hdrs = ["regs.h"], copts = sapi_platform_copts(), deps = [ + ":config", ":syscall", ":violation_cc_proto", "//sandboxed_api/sandbox2/util:strerror", @@ -60,6 +68,7 @@ cc_library( copts = sapi_platform_copts(), visibility = ["//visibility:public"], deps = [ + ":config", ":util", "@com_google_absl//absl/strings", "@com_google_absl//absl/strings:str_format", @@ -73,6 +82,7 @@ cc_test( srcs = ["syscall_test.cc"], copts = sapi_platform_copts(), deps = [ + ":config", ":syscall", "@com_google_absl//absl/strings", "@com_google_googletest//:gtest_main", @@ -85,12 +95,13 @@ cc_library( hdrs = ["result.h"], copts = sapi_platform_copts(), deps = [ + ":config", ":regs", ":syscall", ":util", - "//sandboxed_api/util:statusor", "@com_google_absl//absl/memory", "@com_google_absl//absl/status", + "@com_google_absl//absl/status:statusor", "@com_google_absl//absl/strings", ], ) @@ -272,6 +283,7 @@ cc_library( deps = [ ":client", ":comms", + ":config", ":executor", ":fork_client", ":forkserver_cc_proto", @@ -300,12 +312,12 @@ cc_library( "//sandboxed_api/util:flags", "//sandboxed_api/util:raw_logging", "//sandboxed_api/util:status", - "//sandboxed_api/util:statusor", "@com_google_absl//absl/base:core_headers", "@com_google_absl//absl/container:flat_hash_map", "@com_google_absl//absl/container:flat_hash_set", "@com_google_absl//absl/memory", "@com_google_absl//absl/status", + "@com_google_absl//absl/status:statusor", "@com_google_absl//absl/strings", "@com_google_absl//absl/strings:str_format", "@com_google_absl//absl/synchronization", @@ -374,9 +386,9 @@ cc_library( "//sandboxed_api/sandbox2/util:fileops", "//sandboxed_api/sandbox2/util:strerror", "//sandboxed_api/util:raw_logging", - "//sandboxed_api/util:statusor", "@com_google_absl//absl/memory", "@com_google_absl//absl/status", + "@com_google_absl//absl/status:statusor", "@com_google_absl//absl/strings", "@com_google_absl//absl/strings:str_format", "@org_kernel_libcap//:libcap", @@ -404,6 +416,7 @@ cc_library( hdrs = ["mounts.h"], copts = sapi_platform_copts(), deps = [ + ":config", ":mounttree_cc_proto", "//sandboxed_api/sandbox2/util:file_base", "//sandboxed_api/sandbox2/util:fileops", @@ -411,10 +424,10 @@ cc_library( "//sandboxed_api/sandbox2/util:strerror", "//sandboxed_api/util:raw_logging", "//sandboxed_api/util:status", - "//sandboxed_api/util:statusor", "@com_google_absl//absl/base:core_headers", "@com_google_absl//absl/container:flat_hash_set", "@com_google_absl//absl/status", + "@com_google_absl//absl/status:statusor", "@com_google_absl//absl/strings", "@com_google_protobuf//:protobuf", ], @@ -468,6 +481,7 @@ cc_test( ], deps = [ ":comms", + ":config", ":namespace", ":sandbox2", ":testing", @@ -505,12 +519,13 @@ cc_library( copts = sapi_platform_copts(), visibility = ["//visibility:public"], deps = [ + ":config", "//sandboxed_api/sandbox2/util:file_base", "//sandboxed_api/sandbox2/util:fileops", "//sandboxed_api/sandbox2/util:strerror", "//sandboxed_api/util:raw_logging", - "//sandboxed_api/util:statusor", "@com_google_absl//absl/base:core_headers", + "@com_google_absl//absl/status:statusor", "@com_google_absl//absl/strings", "@com_google_absl//absl/strings:str_format", ], @@ -526,9 +541,9 @@ cc_library( ":util", "//sandboxed_api/sandbox2/util:strerror", "//sandboxed_api/util:status", - "//sandboxed_api/util:statusor", "@com_google_absl//absl/base:core_headers", "@com_google_absl//absl/memory", + "@com_google_absl//absl/status:statusor", "@com_google_absl//absl/strings", ], ) @@ -541,6 +556,7 @@ cc_test( deps = [ ":buffer", ":comms", + ":config", ":sandbox2", ":testing", "//sandboxed_api/util:status_matchers", @@ -573,10 +589,10 @@ cc_library( "//sandboxed_api/util:raw_logging", "//sandboxed_api/util:status", "//sandboxed_api/util:status_proto", - "//sandboxed_api/util:statusor", "@com_google_absl//absl/base:core_headers", "@com_google_absl//absl/memory", "@com_google_absl//absl/status", + "@com_google_absl//absl/status:statusor", "@com_google_absl//absl/strings", "@com_google_absl//absl/strings:str_format", "@com_google_absl//absl/synchronization", @@ -629,6 +645,7 @@ cc_test( copts = sapi_platform_copts(), data = ["//sandboxed_api/sandbox2/testcases:limits"], deps = [ + ":config", ":limits", ":sandbox2", ":testing", @@ -671,6 +688,7 @@ cc_test( "//sandboxed_api/sandbox2/testcases:policy", ], deps = [ + ":config", ":limits", ":regs", ":sandbox2", @@ -695,6 +713,7 @@ cc_test( ], tags = ["local"], deps = [ + ":config", ":sandbox2", ":testing", "//sandboxed_api/sandbox2/util:bpf_helper", @@ -805,6 +824,7 @@ cc_test( "//sandboxed_api/util:status_matchers", "@com_google_absl//absl/memory", "@com_google_absl//absl/status", + "@com_google_absl//absl/status:statusor", "@com_google_absl//absl/strings", "@com_google_glog//:glog", "@com_google_googletest//:gtest_main", diff --git a/sandboxed_api/sandbox2/CMakeLists.txt b/sandboxed_api/sandbox2/CMakeLists.txt index f5919ec..e6a746b 100644 --- a/sandboxed_api/sandbox2/CMakeLists.txt +++ b/sandboxed_api/sandbox2/CMakeLists.txt @@ -17,6 +17,16 @@ add_subdirectory(unwind) add_subdirectory(util) add_subdirectory(network_proxy) +# sandboxed_api/sandbox2:config +add_library(sandbox2_config STATIC + config.h +) +add_library(sandbox2::config ALIAS sandbox2_config) +target_link_libraries(sandbox2_config PRIVATE + absl::config + sapi::base +) + # sandboxed_api/sandbox2:bpfdisassembler add_library(sandbox2_bpfdisassembler STATIC bpfdisassembler.cc @@ -37,6 +47,7 @@ add_library(sandbox2::regs ALIAS sandbox2_regs) target_link_libraries(sandbox2_regs PRIVATE absl::core_headers absl::strings + sandbox2::config sandbox2::strerror sandbox2::syscall sandbox2::violation_proto @@ -72,6 +83,7 @@ target_link_libraries(sandbox2_result PRIVATE absl::base absl::memory absl::strings + sandbox2::config sandbox2::regs sandbox2::syscall sandbox2::util @@ -271,6 +283,8 @@ target_link_libraries(sandbox2_sandbox2 absl::flat_hash_set absl::memory absl::optional + absl::status + absl::statusor absl::str_format absl::strings absl::synchronization @@ -278,6 +292,7 @@ target_link_libraries(sandbox2_sandbox2 sandbox2::bpf_helper sandbox2::client sandbox2::comms + sandbox2::config sandbox2::executor sandbox2::file_base sandbox2::fileops @@ -300,7 +315,6 @@ target_link_libraries(sandbox2_sandbox2 sandbox2::util sandbox2::violation_proto sapi::base - sapi::statusor PUBLIC sapi::flags sapi::status sandbox2::logsink @@ -351,6 +365,8 @@ add_library(sandbox2_forkserver STATIC add_library(sandbox2::forkserver ALIAS sandbox2_forkserver) target_link_libraries(sandbox2_forkserver PRIVATE absl::memory + absl::status + absl::statusor absl::str_format absl::strings libcap::libcap @@ -369,7 +385,6 @@ target_link_libraries(sandbox2_forkserver PRIVATE sandbox2::util sapi::base sapi::raw_logging - sapi::statusor ) # sandboxed_api/sandbox2:fork_client @@ -397,9 +412,11 @@ target_link_libraries(sandbox2_mounts PRIVATE absl::core_headers absl::flat_hash_set absl::status + absl::statusor absl::str_format absl::strings protobuf::libprotobuf + sandbox2::config sandbox2::file_base sandbox2::fileops sandbox2::minielf @@ -408,7 +425,6 @@ target_link_libraries(sandbox2_mounts PRIVATE sapi::base sapi::raw_logging sapi::status - sapi::statusor ) # sandboxed_api/sandbox2:namespace @@ -458,13 +474,14 @@ target_link_libraries(sandbox2_util PRIVATE absl::core_headers absl::str_format absl::strings + sandbox2::config sandbox2::file_base sandbox2::fileops sandbox2::strerror sapi::base sapi::raw_logging - sapi::statusor PUBLIC absl::status + absl::statusor ) target_compile_options(sandbox2_util PRIVATE # The default is 16384, however we need to do a clone with a @@ -482,12 +499,13 @@ add_library(sandbox2::buffer ALIAS sandbox2_buffer) target_link_libraries(sandbox2_buffer PRIVATE absl::core_headers absl::memory + absl::status + absl::statusor absl::strings sandbox2::strerror sandbox2::util sapi::base sapi::status - sapi::statusor ) # sandboxed_api/sandbox2:forkserver_proto @@ -527,6 +545,8 @@ add_library(sandbox2_comms STATIC add_library(sandbox2::comms ALIAS sandbox2_comms) target_link_libraries(sandbox2_comms PRIVATE absl::memory + absl::status + absl::statusor absl::str_format absl::strings sandbox2::strerror @@ -534,7 +554,6 @@ target_link_libraries(sandbox2_comms sapi::base sapi::raw_logging sapi::status_proto - sapi::statusor PUBLIC absl::core_headers absl::status absl::synchronization @@ -566,6 +585,7 @@ if(SAPI_ENABLE_TESTS) ) target_link_libraries(syscall_test PRIVATE absl::strings + sandbox2::config sandbox2::syscall sapi::test_main ) @@ -604,6 +624,7 @@ if(SAPI_ENABLE_TESTS) absl::memory absl::strings sandbox2::comms + sandbox2::config sandbox2::fileops sandbox2::namespace sandbox2::sandbox2 @@ -628,6 +649,7 @@ if(SAPI_ENABLE_TESTS) absl::memory sandbox2::buffer sandbox2::comms + sandbox2::config sandbox2::sandbox2 sandbox2::testing sapi::status_matchers @@ -702,6 +724,7 @@ if(SAPI_ENABLE_TESTS) target_link_libraries(limits_test PRIVATE absl::memory sandbox2::bpf_helper + sandbox2::config sandbox2::limits sandbox2::sandbox2 sandbox2::testing @@ -751,6 +774,7 @@ if(SAPI_ENABLE_TESTS) absl::memory absl::strings sandbox2::bpf_helper + sandbox2::config sandbox2::limits sandbox2::regs sandbox2::sandbox2 @@ -776,6 +800,7 @@ if(SAPI_ENABLE_TESTS) absl::memory absl::strings sandbox2::bpf_helper + sandbox2::config sandbox2::sandbox2 sandbox2::testing sapi::status_matchers @@ -832,6 +857,7 @@ if(SAPI_ENABLE_TESTS) ) target_link_libraries(stack_trace_test PRIVATE absl::memory + absl::status absl::strings sandbox2::bpf_helper sandbox2::fileops diff --git a/sandboxed_api/sandbox2/buffer.cc b/sandboxed_api/sandbox2/buffer.cc index ea5f808..074618d 100644 --- a/sandboxed_api/sandbox2/buffer.cc +++ b/sandboxed_api/sandbox2/buffer.cc @@ -21,6 +21,7 @@ #include #include "absl/memory/memory.h" +#include "absl/status/statusor.h" #include "absl/strings/str_cat.h" #include "sandboxed_api/sandbox2/util.h" #include "sandboxed_api/sandbox2/util/strerror.h" @@ -28,7 +29,7 @@ namespace sandbox2 { // Creates a new Buffer that is backed by the specified file descriptor. -sapi::StatusOr> Buffer::CreateFromFd(int fd) { +absl::StatusOr> Buffer::CreateFromFd(int fd) { auto buffer = absl::WrapUnique(new Buffer{}); struct stat stat_buf; @@ -53,7 +54,7 @@ sapi::StatusOr> Buffer::CreateFromFd(int fd) { // Creates a new Buffer of the specified size, backed by a temporary file that // will be immediately deleted. -sapi::StatusOr> Buffer::CreateWithSize(int64_t size) { +absl::StatusOr> Buffer::CreateWithSize(int64_t size) { int fd; if (!util::CreateMemFd(&fd)) { return absl::InternalError("Could not create buffer temp file"); diff --git a/sandboxed_api/sandbox2/buffer.h b/sandboxed_api/sandbox2/buffer.h index 72f042d..d613bcd 100644 --- a/sandboxed_api/sandbox2/buffer.h +++ b/sandboxed_api/sandbox2/buffer.h @@ -19,7 +19,7 @@ #include #include -#include "sandboxed_api/util/statusor.h" +#include "absl/status/statusor.h" namespace sandbox2 { @@ -37,11 +37,11 @@ class Buffer final { // Creates a new Buffer that is backed by the specified file descriptor. // The Buffer takes ownership of the descriptor and will close it when // destroyed. - static sapi::StatusOr> CreateFromFd(int fd); + static absl::StatusOr> CreateFromFd(int fd); // Creates a new Buffer of the specified size, backed by a temporary file that // will be immediately deleted. - static sapi::StatusOr> CreateWithSize(int64_t size); + static absl::StatusOr> CreateWithSize(int64_t size); // Returns a pointer to the buffer, which is read/write. uint8_t* data() const { return buf_; } diff --git a/sandboxed_api/sandbox2/buffer_test.cc b/sandboxed_api/sandbox2/buffer_test.cc index 78d8cf9..2b72a8b 100644 --- a/sandboxed_api/sandbox2/buffer_test.cc +++ b/sandboxed_api/sandbox2/buffer_test.cc @@ -29,6 +29,7 @@ #include "gtest/gtest.h" #include "absl/memory/memory.h" #include "sandboxed_api/sandbox2/comms.h" +#include "sandboxed_api/sandbox2/config.h" #include "sandboxed_api/sandbox2/executor.h" #include "sandboxed_api/sandbox2/ipc.h" #include "sandboxed_api/sandbox2/policy.h" @@ -83,19 +84,20 @@ std::unique_ptr BufferTestcasePolicy() { .AllowSyscall(__NR_lseek) .AllowSyscall(__NR_close) .BlockSyscallWithErrno(__NR_prlimit64, EPERM) +#ifdef __NR_open .BlockSyscallWithErrno(__NR_open, ENOENT) +#endif .BlockSyscallWithErrno(__NR_openat, ENOENT) +#ifdef __NR_access // On Debian, even static binaries check existence of // /etc/ld.so.nohwcap. .BlockSyscallWithErrno(__NR_access, ENOENT) +#endif +#ifdef __NR_faccessat + .BlockSyscallWithErrno(__NR_faccessat, ENOENT) +#endif .BuildOrDie(); -#if defined(__powerpc64__) - - s2p->AllowUnsafeMmapFiles(); - s2p->AllowUnsafeMmapShared(); -#endif /* defined(__powerpc64__) */ - return s2p; } diff --git a/sandboxed_api/sandbox2/comms.cc b/sandboxed_api/sandbox2/comms.cc index 6bb891c..449db8f 100644 --- a/sandboxed_api/sandbox2/comms.cc +++ b/sandboxed_api/sandbox2/comms.cc @@ -36,6 +36,7 @@ #include "google/protobuf/message.h" #include "absl/memory/memory.h" #include "absl/status/status.h" +#include "absl/status/statusor.h" #include "absl/strings/str_cat.h" #include "absl/strings/str_format.h" #include "absl/synchronization/mutex.h" @@ -44,7 +45,6 @@ #include "sandboxed_api/util/raw_logging.h" #include "sandboxed_api/util/status.h" #include "sandboxed_api/util/status_macros.h" -#include "sandboxed_api/util/statusor.h" #ifdef MEMORY_SANITIZER #include "base/dynamic_annotations.h" @@ -228,7 +228,7 @@ void Comms::Terminate() { } } -bool Comms::SendTLV(uint32_t tag, uint64_t length, const uint8_t* bytes) { +bool Comms::SendTLV(uint32_t tag, size_t length, const void* value) { if (length > GetMaxMsgSize()) { SAPI_RAW_LOG(ERROR, "Maximum TLV message size exceeded: (%u > %u)", length, GetMaxMsgSize()); @@ -249,14 +249,14 @@ bool Comms::SendTLV(uint32_t tag, uint64_t length, const uint8_t* bytes) { length); { absl::MutexLock lock(&tlv_send_transmission_mutex_); - if (!Send(reinterpret_cast(&tag), sizeof(tag))) { + if (!Send(&tag, sizeof(tag))) { return false; } - if (!Send(reinterpret_cast(&length), sizeof(length))) { + if (!Send(&length, sizeof(length))) { return false; } if (length > 0) { - if (!Send(bytes, length)) { + if (!Send(value, length)) { return false; } } @@ -265,40 +265,38 @@ bool Comms::SendTLV(uint32_t tag, uint64_t length, const uint8_t* bytes) { } bool Comms::RecvString(std::string* v) { - TLV tlv; - if (!RecvTLV(&tlv)) { + uint32_t tag; + if (!RecvTLV(&tag, v)) { return false; } - if (tlv.tag != kTagString) { + if (tag != kTagString) { SAPI_RAW_LOG(ERROR, "Expected (kTagString == 0x%x), got: 0x%x", kTagString, - tlv.tag); + tag); return false; } - v->assign(reinterpret_cast(tlv.value.data()), tlv.value.size()); return true; } bool Comms::SendString(const std::string& v) { - return SendTLV(kTagString, v.length(), - reinterpret_cast(v.c_str())); + return SendTLV(kTagString, v.length(), v.c_str()); } bool Comms::RecvBytes(std::vector* buffer) { - TLV tlv; - if (!RecvTLV(&tlv)) { + uint32_t tag; + if (!RecvTLV(&tag, buffer)) { return false; } - if (tlv.tag != kTagBytes) { + if (tag != kTagBytes) { + buffer->clear(); SAPI_RAW_LOG(ERROR, "Expected (kTagBytes == 0x%x), got: 0x%u", kTagBytes, - tlv.tag); + tag); return false; } - buffer->swap(tlv.value); return true; } -bool Comms::SendBytes(const uint8_t* v, uint64_t len) { +bool Comms::SendBytes(const uint8_t* v, size_t len) { return SendTLV(kTagBytes, len, v); } @@ -347,7 +345,7 @@ bool Comms::RecvFD(int* fd) { const auto op = [&msg](int fd) -> ssize_t { PotentiallyBlockingRegion region; - // Use syscall, otherwise we would need to whitelist socketcall() on PPC. + // Use syscall, otherwise we would need to allow socketcall() on PPC. return TEMP_FAILURE_RETRY( util::Syscall(__NR_recvmsg, fd, reinterpret_cast(&msg), 0)); }; @@ -419,7 +417,7 @@ bool Comms::SendFD(int fd) { int* fds = reinterpret_cast(CMSG_DATA(cmsg)); fds[0] = fd; - InternalTLV tlv = {kTagFd, sizeof(tlv.val), 0}; + InternalTLV tlv = {kTagFd, 0}; iovec iov; iov.iov_base = &tlv; @@ -462,8 +460,9 @@ bool Comms::SendFD(int fd) { } bool Comms::RecvProtoBuf(google::protobuf::Message* message) { - TLV tlv; - if (!RecvTLV(&tlv)) { + uint32_t tag; + std::vector bytes; + if (!RecvTLV(&tag, &bytes)) { if (IsConnected()) { SAPI_RAW_PLOG(ERROR, "RecvProtoBuf failed for (%s)", socket_name_); } else { @@ -473,11 +472,11 @@ bool Comms::RecvProtoBuf(google::protobuf::Message* message) { return false; } - if (tlv.tag != kTagProto2) { - SAPI_RAW_LOG(ERROR, "Expected tag: 0x%x, got: 0x%u", kTagProto2, tlv.tag); + if (tag != kTagProto2) { + SAPI_RAW_LOG(ERROR, "Expected tag: 0x%x, got: 0x%u", kTagProto2, tag); return false; } - return message->ParseFromArray(tlv.value.data(), tlv.value.size()); + return message->ParseFromArray(bytes.data(), bytes.size()); } bool Comms::SendProtoBuf(const google::protobuf::Message& message) { @@ -513,8 +512,9 @@ socklen_t Comms::CreateSockaddrUn(sockaddr_un* sun) { return slen; } -bool Comms::Send(const uint8_t* bytes, uint64_t len) { - uint64_t total_sent = 0; +bool Comms::Send(const void* data, size_t len) { + size_t total_sent = 0; + const char* bytes = reinterpret_cast(data); const auto op = [bytes, len, &total_sent](int fd) -> ssize_t { PotentiallyBlockingRegion region; return TEMP_FAILURE_RETRY(write(fd, &bytes[total_sent], len - total_sent)); @@ -545,8 +545,9 @@ bool Comms::Send(const uint8_t* bytes, uint64_t len) { return true; } -bool Comms::Recv(uint8_t* bytes, uint64_t len) { - uint64_t total_recv = 0; +bool Comms::Recv(void* data, size_t len) { + size_t total_recv = 0; + char* bytes = reinterpret_cast(data); const auto op = [bytes, len, &total_recv](int fd) -> ssize_t { PotentiallyBlockingRegion region; return TEMP_FAILURE_RETRY(read(fd, &bytes[total_recv], len - total_recv)); @@ -573,7 +574,7 @@ bool Comms::Recv(uint8_t* bytes, uint64_t len) { } // Internal helper method (low level). -bool Comms::RecvTL(uint32_t* tag, uint64_t* length) { +bool Comms::RecvTL(uint32_t* tag, size_t* length) { if (!Recv(reinterpret_cast(tag), sizeof(*tag))) { return false; } @@ -599,50 +600,49 @@ bool Comms::RecvTL(uint32_t* tag, uint64_t* length) { return true; } -bool Comms::RecvTLV(TLV* tlv) { - absl::MutexLock lock(&tlv_recv_transmission_mutex_); - uint64_t length; - if (!RecvTL(&tlv->tag, &length)) { - return false; - } - - tlv->value.resize(length); - return length == 0 || Recv(tlv->value.data(), length); +bool Comms::RecvTLV(uint32_t* tag, std::vector* value) { + return RecvTLVGeneric(tag, value); } -bool Comms::RecvTLV(uint32_t* tag, std::vector* value) { +bool Comms::RecvTLV(uint32_t* tag, std::string* value) { + return RecvTLVGeneric(tag, value); +} + +template +bool Comms::RecvTLVGeneric(uint32_t* tag, T* value) { absl::MutexLock lock(&tlv_recv_transmission_mutex_); - uint64_t length; + size_t length; if (!RecvTL(tag, &length)) { return false; } value->resize(length); - return length == 0 || Recv(value->data(), length); + return length == 0 || Recv(reinterpret_cast(value->data()), length); } -bool Comms::RecvTLV(uint32_t* tag, uint64_t* length, void* buffer, - uint64_t buffer_size) { +bool Comms::RecvTLV(uint32_t* tag, size_t* length, void* buffer, + size_t buffer_size) { absl::MutexLock lock(&tlv_recv_transmission_mutex_); if (!RecvTL(tag, length)) { return false; } + if (*length == 0) { + return true; + } + if (*length > buffer_size) { SAPI_RAW_LOG(ERROR, "Buffer size too small (0x%x > 0x%x)", *length, buffer_size); return false; - } else if (*length > 0) { - if (!Recv(reinterpret_cast(buffer), *length)) { - return false; - } } - return true; + + return Recv(reinterpret_cast(buffer), *length); } -bool Comms::RecvInt(void* buffer, uint64_t len, uint32_t tag) { +bool Comms::RecvInt(void* buffer, size_t len, uint32_t tag) { uint32_t received_tag; - uint64_t received_length; + size_t received_length; if (!RecvTLV(&received_tag, &received_length, buffer, len)) { return false; } diff --git a/sandboxed_api/sandbox2/comms.h b/sandboxed_api/sandbox2/comms.h index c4ab84e..fca8c53 100644 --- a/sandboxed_api/sandbox2/comms.h +++ b/sandboxed_api/sandbox2/comms.h @@ -60,7 +60,7 @@ class Comms { static constexpr uint32_t kTagFd = 0X80000201; // Any payload size above this limit will LOG(WARNING). - static constexpr uint64_t kWarnMsgSize = (256ULL << 20); + static constexpr size_t kWarnMsgSize = (256ULL << 20); // Sandbox2-specific convention where FD=1023 is always passed to the // sandboxed process as a communication channel (encapsulated in the @@ -103,14 +103,17 @@ class Comms { // Note: The actual size is "unlimited", although the Buffer API is more // efficient for large transfers. There is an arbitrary limit to ~2GiB to // avoid protobuf serialization issues. - uint64_t GetMaxMsgSize() const { return std::numeric_limits::max(); } + size_t GetMaxMsgSize() const { return std::numeric_limits::max(); } - bool SendTLV(uint32_t tag, uint64_t length, const uint8_t* bytes); + bool SendTLV(uint32_t tag, size_t length, const void* value); // Receive a TLV structure, the memory for the value will be allocated // by std::vector. bool RecvTLV(uint32_t* tag, std::vector* value); + // Receive a TLV structure, the memory for the value will be allocated + // by std::string. + bool RecvTLV(uint32_t* tag, std::string* value); // Receives a TLV value into a specified buffer without allocating memory. - bool RecvTLV(uint32_t* tag, uint64_t* length, void* buffer, uint64_t buffer_size); + bool RecvTLV(uint32_t* tag, size_t* length, void* buffer, size_t buffer_size); // Sends/receives various types of data. bool RecvUint8(uint8_t* v) { return RecvIntGeneric(v, kTagUint8); } @@ -135,7 +138,7 @@ class Comms { bool SendString(const std::string& v); bool RecvBytes(std::vector* buffer); - bool SendBytes(const uint8_t* v, uint64_t len); + bool SendBytes(const uint8_t* v, size_t len); bool SendBytes(const std::vector& buffer); // Receives remote process credentials. @@ -174,38 +177,32 @@ class Comms { // State of the channel (enum), socket will have to be connected later on. State state_ = State::kUnconnected; - // TLV structure used to pass messages around. - struct TLV { - uint32_t tag; - std::vector value; - }; - // Special struct for passing credentials or FDs. Different from the one above // as it inlines the value. This is important as the data is transmitted using // sendmsg/recvmsg instead of send/recv. struct ABSL_ATTRIBUTE_PACKED InternalTLV { uint32_t tag; uint32_t len; - uint64_t val; }; // Fills sockaddr_un struct with proper values. socklen_t CreateSockaddrUn(sockaddr_un* sun); // Support for EINTR and size completion. - bool Send(const uint8_t* bytes, uint64_t len); - bool Recv(uint8_t* bytes, uint64_t len); + bool Send(const void* data, size_t len); + bool Recv(void* data, size_t len); // Receives tag and length. Assumes that the `tlv_transmission_mutex_` mutex // is locked. - bool RecvTL(uint32_t* tag, uint64_t* length) + bool RecvTL(uint32_t* tag, size_t* length) ABSL_EXCLUSIVE_LOCKS_REQUIRED(tlv_recv_transmission_mutex_); - // Receives whole TLV structure, allocates memory for the data. - bool RecvTLV(TLV* tlv); + // T has to be a ContiguousContainer + template + bool RecvTLVGeneric(uint32_t* tag, T* value); // Receives arbitrary integers. - bool RecvInt(void* buffer, uint64_t len, uint32_t tag); + bool RecvInt(void* buffer, size_t len, uint32_t tag); template bool RecvIntGeneric(T* output, uint32_t tag) { @@ -214,7 +211,7 @@ class Comms { template bool SendGeneric(T value, uint32_t tag) { - return SendTLV(tag, sizeof(T), reinterpret_cast(&value)); + return SendTLV(tag, sizeof(T), &value); } }; diff --git a/sandboxed_api/sandbox2/config.h b/sandboxed_api/sandbox2/config.h new file mode 100644 index 0000000..209cd82 --- /dev/null +++ b/sandboxed_api/sandbox2/config.h @@ -0,0 +1,85 @@ +// Copyright 2020 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef SANDBOXED_API_SANDBOX2_CONFIG_H_ +#define SANDBOXED_API_SANDBOX2_CONFIG_H_ + +#include + +#include "absl/base/config.h" + +// GCC/Clang define __x86_64__, Visual Studio uses _M_X64 +#if defined(__x86_64__) || defined(_M_X64) +#define SAPI_X86_64 1 + +// Check various spellings for 64-bit POWER. Not checking for Visual Studio, as +// it does not support 64-bit POWER. +#elif (defined(__PPC64__) || defined(__powerpc64__) || defined(__ppc64__)) && \ + defined(ABSL_IS_LITTLE_ENDIAN) +#define SAPI_PPC64_LE 1 + +// Spellings for AArch64 +#elif defined(__aarch64__) || defined(_M_ARM64) +#define SAPI_ARM64 1 +#endif + +namespace sandbox2 { + +namespace cpu { + +// CPU architectures known to Sandbox2 +enum Architecture : uint16_t { + // Linux: Use a magic value, so it can be easily spotted in the seccomp-bpf + // bytecode decompilation stream. Must be < (1<<15), as/ that's the size of + // data which can be returned by BPF. + kUnknown = 0xCAF0, + kX8664, + kX86, + kPPC64LE, + kArm64, +}; + +} // namespace cpu + +namespace host_cpu { + +// Returns the current host CPU architecture if supported. If not supported, +// returns cpu::kUnknown. +constexpr cpu::Architecture Architecture() { +#if defined(SAPI_X86_64) + return cpu::kX8664; +#elif defined(SAPI_PPC64_LE) + return cpu::kPPC64LE; +#elif defined(SAPI_ARM64) + return cpu::kArm64; +#else + return cpu::kUnknown; +#endif +} + +constexpr bool IsX8664() { return Architecture() == cpu::kX8664; } + +constexpr bool IsPPC64LE() { return Architecture() == cpu::kPPC64LE; } + +constexpr bool IsArm64() { return Architecture() == cpu::kArm64; } + +} // namespace host_cpu + +static_assert(host_cpu::Architecture() != cpu::kUnknown, + "Host CPU architecture is not supported: One of x86-64, POWER64 " + "(little endian) or AArch64 is required."); + +} // namespace sandbox2 + +#endif // SANDBOXED_API_SANDBOX2_CONFIG_H_ diff --git a/sandboxed_api/sandbox2/examples/network_proxy/BUILD.bazel b/sandboxed_api/sandbox2/examples/network_proxy/BUILD.bazel index 133cb88..ccbefc5 100644 --- a/sandboxed_api/sandbox2/examples/network_proxy/BUILD.bazel +++ b/sandboxed_api/sandbox2/examples/network_proxy/BUILD.bazel @@ -27,7 +27,6 @@ cc_binary( deps = [ "//sandboxed_api/sandbox2", "//sandboxed_api/sandbox2:comms", - "//sandboxed_api/sandbox2/network_proxy:filtering", "//sandboxed_api/sandbox2/util:bpf_helper", "//sandboxed_api/sandbox2/util:fileops", "//sandboxed_api/sandbox2/util:runfiles", @@ -48,7 +47,9 @@ cc_binary( "//sandboxed_api/sandbox2/util:strerror", "//sandboxed_api/util:flags", "//sandboxed_api/util:status", - "//sandboxed_api/util:statusor", + "@com_google_absl//absl/status", + "@com_google_absl//absl/status:statusor", + "@com_google_absl//absl/strings", "@com_google_absl//absl/strings:str_format", ], ) diff --git a/sandboxed_api/sandbox2/examples/network_proxy/CMakeLists.txt b/sandboxed_api/sandbox2/examples/network_proxy/CMakeLists.txt index 084c8ed..3df9f21 100644 --- a/sandboxed_api/sandbox2/examples/network_proxy/CMakeLists.txt +++ b/sandboxed_api/sandbox2/examples/network_proxy/CMakeLists.txt @@ -31,13 +31,14 @@ target_link_libraries(sandbox2_networkproxy_sandbox PRIVATE sapi::flags ) - # sandboxed_api/sandbox2/examples/networkproxy:networkproxy_bin add_executable(sandbox2_networkproxy_bin networkproxy_bin.cc ) add_executable(sandbox2::networkproxy_bin ALIAS sandbox2_networkproxy_bin) target_link_libraries(sandbox2_networkproxy_bin PRIVATE + absl::status + absl::statusor absl::str_format glog::glog gflags::gflags @@ -48,6 +49,4 @@ target_link_libraries(sandbox2_networkproxy_bin PRIVATE sapi::base sapi::flags sapi::status - sapi::statusor ) - diff --git a/sandboxed_api/sandbox2/examples/network_proxy/networkproxy_bin.cc b/sandboxed_api/sandbox2/examples/network_proxy/networkproxy_bin.cc index c22947d..d5d01f0 100644 --- a/sandboxed_api/sandbox2/examples/network_proxy/networkproxy_bin.cc +++ b/sandboxed_api/sandbox2/examples/network_proxy/networkproxy_bin.cc @@ -12,15 +12,16 @@ #include #include "sandboxed_api/util/flag.h" +#include "absl/status/status.h" +#include "absl/status/statusor.h" +#include "absl/strings/str_cat.h" #include "absl/strings/str_format.h" #include "sandboxed_api/sandbox2/client.h" #include "sandboxed_api/sandbox2/comms.h" #include "sandboxed_api/sandbox2/network_proxy/client.h" #include "sandboxed_api/sandbox2/util/fileops.h" #include "sandboxed_api/sandbox2/util/strerror.h" -#include "sandboxed_api/util/status.h" #include "sandboxed_api/util/status_macros.h" -#include "sandboxed_api/util/statusor.h" ABSL_FLAG(bool, connect_with_handler, true, "Connect using automatic mode."); @@ -58,7 +59,7 @@ absl::Status CommunicationTest(int sock) { return absl::OkStatus(); } -sapi::StatusOr CreateAddres(int port) { +absl::StatusOr CreateAddres(int port) { static struct sockaddr_in6 saddr {}; saddr.sin6_family = AF_INET6; saddr.sin6_port = htons(port); @@ -86,7 +87,7 @@ absl::Status ConnectWithHandler(int s, const struct sockaddr_in6& saddr) { return absl::OkStatus(); } -sapi::StatusOr ConnectToServer(int port) { +absl::StatusOr ConnectToServer(int port) { SAPI_ASSIGN_OR_RETURN(struct sockaddr_in6 saddr, CreateAddres(port)); sandbox2::file_util::fileops::FDCloser s(socket(AF_INET6, SOCK_STREAM, 0)); @@ -134,7 +135,7 @@ int main(int argc, char** argv) { return 2; } - sapi::StatusOr sock_s = ConnectToServer(port); + absl::StatusOr sock_s = ConnectToServer(port); if (!sock_s.ok()) { LOG(ERROR) << sock_s.status().message(); return 3; diff --git a/sandboxed_api/sandbox2/examples/static/static_sandbox.cc b/sandboxed_api/sandbox2/examples/static/static_sandbox.cc index 44f794c..7290b72 100644 --- a/sandboxed_api/sandbox2/examples/static/static_sandbox.cc +++ b/sandboxed_api/sandbox2/examples/static/static_sandbox.cc @@ -52,8 +52,10 @@ std::unique_ptr GetPolicy() { // Allow the getpid() syscall. .AllowSyscall(__NR_getpid) +#ifdef __NR_access // On Debian, even static binaries check existence of /etc/ld.so.nohwcap. .BlockSyscallWithErrno(__NR_access, ENOENT) +#endif // Examples for AddPolicyOnSyscall: .AddPolicyOnSyscall(__NR_write, diff --git a/sandboxed_api/sandbox2/executor.cc b/sandboxed_api/sandbox2/executor.cc index 6170a9d..f2afb3a 100644 --- a/sandboxed_api/sandbox2/executor.cc +++ b/sandboxed_api/sandbox2/executor.cc @@ -86,7 +86,7 @@ pid_t Executor::StartSubProcess(int32_t clone_flags, const Namespace* ns, if (!path_.empty()) { exec_fd_ = open(path_.c_str(), O_PATH); if (exec_fd_ < 0) { - LOG(ERROR) << "Could not open file " << path_; + PLOG(ERROR) << "Could not open file " << path_; return -1; } } diff --git a/sandboxed_api/sandbox2/forkserver.cc b/sandboxed_api/sandbox2/forkserver.cc index 2125927..dc6a283 100644 --- a/sandboxed_api/sandbox2/forkserver.cc +++ b/sandboxed_api/sandbox2/forkserver.cc @@ -36,6 +36,7 @@ #include "absl/memory/memory.h" #include "absl/status/status.h" +#include "absl/status/statusor.h" #include "absl/strings/match.h" #include "absl/strings/str_cat.h" #include "absl/strings/str_format.h" @@ -55,7 +56,6 @@ #include "sandboxed_api/sandbox2/util/fileops.h" #include "sandboxed_api/sandbox2/util/strerror.h" #include "sandboxed_api/util/raw_logging.h" -#include "sandboxed_api/util/statusor.h" namespace { // "Moves" the old FD to the new FD number. @@ -142,7 +142,7 @@ absl::Status SendPid(int signaling_fd) { return absl::OkStatus(); } -sapi::StatusOr ReceivePid(int signaling_fd) { +absl::StatusOr ReceivePid(int signaling_fd) { union { struct cmsghdr cmh; char ctrl[CMSG_SPACE(sizeof(struct ucred))]; diff --git a/sandboxed_api/sandbox2/limits_test.cc b/sandboxed_api/sandbox2/limits_test.cc index 1f3386a..35d9bba 100644 --- a/sandboxed_api/sandbox2/limits_test.cc +++ b/sandboxed_api/sandbox2/limits_test.cc @@ -23,6 +23,7 @@ #include "gmock/gmock.h" #include "gtest/gtest.h" #include "absl/memory/memory.h" +#include "sandboxed_api/sandbox2/config.h" #include "sandboxed_api/sandbox2/executor.h" #include "sandboxed_api/sandbox2/policy.h" #include "sandboxed_api/sandbox2/policybuilder.h" diff --git a/sandboxed_api/sandbox2/monitor.cc b/sandboxed_api/sandbox2/monitor.cc index 10e0710..d15c1af 100644 --- a/sandboxed_api/sandbox2/monitor.cc +++ b/sandboxed_api/sandbox2/monitor.cc @@ -49,6 +49,7 @@ #include "absl/time/time.h" #include "sandboxed_api/sandbox2/client.h" #include "sandboxed_api/sandbox2/comms.h" +#include "sandboxed_api/sandbox2/config.h" #include "sandboxed_api/sandbox2/executor.h" #include "sandboxed_api/sandbox2/limits.h" #include "sandboxed_api/sandbox2/mounts.h" @@ -761,7 +762,7 @@ void Monitor::LogSyscallViolation(const Syscall& syscall) const { void Monitor::EventPtraceSeccomp(pid_t pid, int event_msg) { // If the seccomp-policy is using RET_TRACE, we request that it returns the // syscall architecture identifier in the SECCOMP_RET_DATA. - const auto syscall_arch = static_cast(event_msg); + const auto syscall_arch = static_cast(event_msg); Regs regs(pid); auto status = regs.Fetch(); if (!status.ok()) { diff --git a/sandboxed_api/sandbox2/mounts.cc b/sandboxed_api/sandbox2/mounts.cc index 2f734e2..4c71d52 100644 --- a/sandboxed_api/sandbox2/mounts.cc +++ b/sandboxed_api/sandbox2/mounts.cc @@ -27,19 +27,20 @@ #include "google/protobuf/util/message_differencer.h" #include "absl/container/flat_hash_set.h" +#include "absl/status/statusor.h" #include "absl/strings/ascii.h" #include "absl/strings/match.h" #include "absl/strings/str_cat.h" #include "absl/strings/str_join.h" #include "absl/strings/str_split.h" #include "absl/strings/string_view.h" +#include "sandboxed_api/sandbox2/config.h" #include "sandboxed_api/sandbox2/util/fileops.h" #include "sandboxed_api/sandbox2/util/minielf.h" #include "sandboxed_api/sandbox2/util/path.h" #include "sandboxed_api/sandbox2/util/strerror.h" #include "sandboxed_api/util/raw_logging.h" #include "sandboxed_api/util/status_macros.h" -#include "sandboxed_api/util/statusor.h" namespace sandbox2 { namespace { @@ -97,7 +98,7 @@ absl::string_view GetOutsidePath(const MountTree::Node& node) { } } -sapi::StatusOr ExistingPathInsideDir( +absl::StatusOr ExistingPathInsideDir( absl::string_view dir_path, absl::string_view relative_path) { auto path = file::CleanPath(file::JoinPath(dir_path, relative_path)); if (file_util::fileops::StripBasename(path) != dir_path) { @@ -112,6 +113,8 @@ sapi::StatusOr ExistingPathInsideDir( absl::Status ValidateInterpreter(absl::string_view interpreter) { const absl::flat_hash_set allowed_interpreters = { "/lib64/ld-linux-x86-64.so.2", + "/lib64/ld64.so.2", // PPC64 + "/lib/ld-linux-aarch64.so.1", // AArch64 }; if (!allowed_interpreters.contains(interpreter)) { @@ -132,15 +135,21 @@ std::string ResolveLibraryPath(absl::string_view lib_name, return ""; } +constexpr absl::string_view GetPlatformCPUName() { + switch (host_cpu::Architecture()) { + case cpu::kX8664: + return "x86_64"; + case cpu::kPPC64LE: + return "ppc64"; + case cpu::kArm64: + return "aarch64"; + default: + return "unknown"; + } +} + std::string GetPlatform(absl::string_view interpreter) { -#if defined(__x86_64__) - constexpr absl::string_view kCpuPlatform = "x86_64"; -#elif defined(__powerpc64__) - constexpr absl::string_view kCpuPlatform = "ppc64"; -#else - constexpr absl::string_view kCpuPlatform = "unknown"; -#endif - return absl::StrCat(kCpuPlatform, "-linux-gnu"); + return absl::StrCat(GetPlatformCPUName(), "-linux-gnu"); } } // namespace @@ -496,7 +505,7 @@ std::string MountFlagsToString(uint64_t flags) { SAPI_MAP(MS_POSIXACL), SAPI_MAP(MS_UNBINDABLE), SAPI_MAP(MS_PRIVATE), - SAPI_MAP(MS_SLAVE), + SAPI_MAP(MS_SLAVE), // Inclusive language: system constant SAPI_MAP(MS_SHARED), SAPI_MAP(MS_RELATIME), SAPI_MAP(MS_KERNMOUNT), diff --git a/sandboxed_api/sandbox2/namespace_test.cc b/sandboxed_api/sandbox2/namespace_test.cc index 89da97b..2bedf16 100644 --- a/sandboxed_api/sandbox2/namespace_test.cc +++ b/sandboxed_api/sandbox2/namespace_test.cc @@ -28,6 +28,7 @@ #include "absl/strings/numbers.h" #include "absl/strings/str_cat.h" #include "sandboxed_api/sandbox2/comms.h" +#include "sandboxed_api/sandbox2/config.h" #include "sandboxed_api/sandbox2/executor.h" #include "sandboxed_api/sandbox2/policy.h" #include "sandboxed_api/sandbox2/policybuilder.h" diff --git a/sandboxed_api/sandbox2/network_proxy/BUILD.bazel b/sandboxed_api/sandbox2/network_proxy/BUILD.bazel index 3a3a9d6..ac58976 100644 --- a/sandboxed_api/sandbox2/network_proxy/BUILD.bazel +++ b/sandboxed_api/sandbox2/network_proxy/BUILD.bazel @@ -30,6 +30,7 @@ cc_library( "//sandboxed_api/sandbox2:comms", "//sandboxed_api/sandbox2/util:fileops", "@com_google_absl//absl/memory", + "@com_google_absl//absl/status:statusor", "@com_google_absl//absl/strings", "@com_google_glog//:glog", ], @@ -43,9 +44,11 @@ cc_library( visibility = ["//visibility:public"], deps = [ "//sandboxed_api/sandbox2:comms", + "//sandboxed_api/sandbox2:config", "//sandboxed_api/sandbox2/util:strerror", "//sandboxed_api/util:status", "@com_google_absl//absl/memory", + "@com_google_absl//absl/status", "@com_google_absl//absl/strings", "@com_google_absl//absl/synchronization", "@com_google_glog//:glog", @@ -61,7 +64,8 @@ cc_library( "//sandboxed_api/sandbox2:comms", "//sandboxed_api/sandbox2/util:strerror", "//sandboxed_api/util:status", - "//sandboxed_api/util:statusor", + "@com_google_absl//absl/status", + "@com_google_absl//absl/status:statusor", "@com_google_absl//absl/strings", "@com_google_glog//:glog", ], diff --git a/sandboxed_api/sandbox2/network_proxy/CMakeLists.txt b/sandboxed_api/sandbox2/network_proxy/CMakeLists.txt index 8ae6c40..781cb9d 100644 --- a/sandboxed_api/sandbox2/network_proxy/CMakeLists.txt +++ b/sandboxed_api/sandbox2/network_proxy/CMakeLists.txt @@ -33,14 +33,14 @@ add_library(sandbox2_network_proxy_filtering STATIC filtering.h ) add_library(sandbox2::network_proxy_filtering ALIAS sandbox2_network_proxy_filtering) -target_link_libraries(sandbox2_network_proxy_filtering PRIVATE - absl::memory - glog::glog - sandbox2::comms - sandbox2::fileops - sapi::base +target_link_libraries(sandbox2_network_proxy_filtering + PRIVATE absl::memory + absl::status + glog::glog + sandbox2::comms + sandbox2::fileops + sapi::base PUBLIC sapi::status - sapi::statusor ) # sandboxed_api/sandbox2/network_proxy:client @@ -54,6 +54,7 @@ target_link_libraries(sandbox2_network_proxy_client PRIVATE absl::synchronization glog::glog sandbox2::comms + sandbox2::config sandbox2::strerror sapi::base sapi::status diff --git a/sandboxed_api/sandbox2/network_proxy/client.cc b/sandboxed_api/sandbox2/network_proxy/client.cc index 586d201..3dde8c1 100644 --- a/sandboxed_api/sandbox2/network_proxy/client.cc +++ b/sandboxed_api/sandbox2/network_proxy/client.cc @@ -25,9 +25,10 @@ #include #include "absl/memory/memory.h" +#include "absl/status/status.h" #include "absl/strings/str_cat.h" +#include "sandboxed_api/sandbox2/config.h" #include "sandboxed_api/sandbox2/util/strerror.h" -#include "sandboxed_api/util/status.h" #include "sandboxed_api/util/status_macros.h" namespace sandbox2 { @@ -36,23 +37,26 @@ namespace sandbox2 { constexpr int SYS_SECCOMP = 1; #endif -#if defined(__x86_64__) +#if defined(SAPI_X86_64) constexpr int kRegResult = REG_RAX; constexpr int kRegSyscall = REG_RAX; constexpr int kRegArg0 = REG_RDI; constexpr int kRegArg1 = REG_RSI; constexpr int kRegArg2 = REG_RDX; -#endif -#if defined(__powerpc64__) +#elif defined(SAPI_PPC64_LE) constexpr int kRegResult = 3; constexpr int kRegSyscall = 0; constexpr int kRegArg0 = 3; constexpr int kRegArg1 = 4; constexpr int kRegArg2 = 5; +#elif defined(SAPI_ARM64) +constexpr int kRegResult = 0; +constexpr int kRegSyscall = 8; +constexpr int kRegArg0 = 0; +constexpr int kRegArg1 = 1; +constexpr int kRegArg2 = 2; #endif -constexpr char NetworkProxyClient::kFDName[]; - int NetworkProxyClient::ConnectHandler(int sockfd, const struct sockaddr* addr, socklen_t addrlen) { absl::Status status = Connect(sockfd, addr, addrlen); @@ -154,20 +158,22 @@ void NetworkProxyHandler::InvokeOldAct(int nr, siginfo_t* info, void NetworkProxyHandler::ProcessSeccompTrap(int nr, siginfo_t* info, void* void_context) { - ucontext_t* ctx = (ucontext_t*)(void_context); if (info->si_code != SYS_SECCOMP) { InvokeOldAct(nr, info, void_context); return; } - if (!ctx) return; + auto* ctx = static_cast(void_context); + if (!ctx) { + return; + } -#if defined(__x86_64__) +#if defined(SAPI_X86_64) auto* registers = ctx->uc_mcontext.gregs; -#elif defined(__powerpc64__) +#elif defined(SAPI_PPC64_LE) auto* registers = ctx->uc_mcontext.gp_regs; - using ppc_gpreg_t = std::decay::type; +#elif defined(SAPI_ARM64) + auto* registers = ctx->uc_mcontext.regs; #endif - int syscall = registers[kRegSyscall]; int sockfd; @@ -178,14 +184,13 @@ void NetworkProxyHandler::ProcessSeccompTrap(int nr, siginfo_t* info, sockfd = static_cast(registers[kRegArg0]); addr = reinterpret_cast(registers[kRegArg1]); addrlen = static_cast(registers[kRegArg2]); -#if defined(__powerpc64__) +#if defined(SAPI_PPC64_LE) } else if (syscall == __NR_socketcall && static_cast(registers[kRegArg0]) == SYS_CONNECT) { - ppc_gpreg_t* args = reinterpret_cast(registers[kRegArg1]); - - sockfd = static_cast(args[0]); - addr = reinterpret_cast(args[1]); - addrlen = static_cast(args[2]); + auto* connect_args = reinterpret_cast(registers[kRegArg1]); + sockfd = static_cast(connect_args[0]); + addr = reinterpret_cast(connect_args[1]); + addrlen = static_cast(connect_args[2]); #endif } else { InvokeOldAct(nr, info, void_context); diff --git a/sandboxed_api/sandbox2/network_proxy/client.h b/sandboxed_api/sandbox2/network_proxy/client.h index e7af4dc..7318993 100644 --- a/sandboxed_api/sandbox2/network_proxy/client.h +++ b/sandboxed_api/sandbox2/network_proxy/client.h @@ -17,9 +17,9 @@ #include +#include "absl/status/status.h" #include "absl/synchronization/mutex.h" #include "sandboxed_api/sandbox2/comms.h" -#include "sandboxed_api/util/status.h" namespace sandbox2 { diff --git a/sandboxed_api/sandbox2/network_proxy/filtering.cc b/sandboxed_api/sandbox2/network_proxy/filtering.cc index d6389d8..aac47df 100644 --- a/sandboxed_api/sandbox2/network_proxy/filtering.cc +++ b/sandboxed_api/sandbox2/network_proxy/filtering.cc @@ -17,16 +17,17 @@ #include #include +#include "absl/status/status.h" +#include "absl/status/statusor.h" #include "absl/strings/numbers.h" #include "absl/strings/str_cat.h" #include "absl/strings/str_split.h" #include "sandboxed_api/sandbox2/util/strerror.h" -#include "sandboxed_api/util/status.h" #include "sandboxed_api/util/status_macros.h" namespace sandbox2 { -static sapi::StatusOr Addr6ToString( +static absl::StatusOr Addr6ToString( const struct sockaddr_in6* saddr) { char addr[INET6_ADDRSTRLEN]; int port = htons(saddr->sin6_port); @@ -38,7 +39,7 @@ static sapi::StatusOr Addr6ToString( } // Converts sockaddr_in structure into a string IPv4 representation. -static sapi::StatusOr Addr4ToString( +static absl::StatusOr Addr4ToString( const struct sockaddr_in* saddr) { char addr[INET_ADDRSTRLEN]; int port = htons(saddr->sin_port); @@ -50,7 +51,7 @@ static sapi::StatusOr Addr4ToString( } // Converts sockaddr_in6 structure into a string IPv6 representation. -sapi::StatusOr AddrToString(const struct sockaddr* saddr) { +absl::StatusOr AddrToString(const struct sockaddr* saddr) { switch (saddr->sa_family) { case AF_INET: return Addr4ToString(reinterpret_cast(saddr)); diff --git a/sandboxed_api/sandbox2/network_proxy/filtering.h b/sandboxed_api/sandbox2/network_proxy/filtering.h index fb6714a..b830d8e 100644 --- a/sandboxed_api/sandbox2/network_proxy/filtering.h +++ b/sandboxed_api/sandbox2/network_proxy/filtering.h @@ -19,14 +19,14 @@ #include +#include "absl/status/statusor.h" #include "sandboxed_api/sandbox2/comms.h" -#include "sandboxed_api/util/statusor.h" namespace sandbox2 { // Converts sockaddr_in or sockaddr_in6 structure into a string // representation. -sapi::StatusOr AddrToString(const struct sockaddr* saddr); +absl::StatusOr AddrToString(const struct sockaddr* saddr); struct IPv4 { in_addr_t ip; diff --git a/sandboxed_api/sandbox2/network_proxy/server.cc b/sandboxed_api/sandbox2/network_proxy/server.cc index c183812..6ba1ba7 100644 --- a/sandboxed_api/sandbox2/network_proxy/server.cc +++ b/sandboxed_api/sandbox2/network_proxy/server.cc @@ -26,6 +26,7 @@ #include #include "absl/memory/memory.h" +#include "absl/status/statusor.h" #include "sandboxed_api/sandbox2/util/fileops.h" namespace sandbox2 { @@ -104,7 +105,7 @@ void NetworkProxyServer::NotifySuccess() { } void NetworkProxyServer::NotifyViolation(const struct sockaddr* saddr) { - if (sapi::StatusOr result = AddrToString(saddr); result.ok()) { + if (absl::StatusOr result = AddrToString(saddr); result.ok()) { violation_msg_ = std::move(result).value(); } else { violation_msg_ = std::string(result.status().message()); diff --git a/sandboxed_api/sandbox2/notify_test.cc b/sandboxed_api/sandbox2/notify_test.cc index 77fc67f..8e0bc38 100644 --- a/sandboxed_api/sandbox2/notify_test.cc +++ b/sandboxed_api/sandbox2/notify_test.cc @@ -49,8 +49,12 @@ std::unique_ptr NotifyTestcasePolicy() { .AllowWrite() .AllowSyscall(__NR_close) .AddPolicyOnSyscall(__NR_personality, {SANDBOX2_TRACE}) +#ifdef __NR_open .BlockSyscallWithErrno(__NR_open, ENOENT) +#endif +#ifdef __NR_access .BlockSyscallWithErrno(__NR_access, ENOENT) +#endif .BlockSyscallWithErrno(__NR_openat, ENOENT) .BlockSyscallWithErrno(__NR_prlimit64, EPERM) .BuildOrDie(); diff --git a/sandboxed_api/sandbox2/policy.cc b/sandboxed_api/sandbox2/policy.cc index e631c0a..398fbfe 100644 --- a/sandboxed_api/sandbox2/policy.cc +++ b/sandboxed_api/sandbox2/policy.cc @@ -82,41 +82,41 @@ std::vector Policy::GetDefaultPolicy() const { bpf_labels l = {0}; std::vector policy = { - // If compiled arch is different than the runtime one, inform the Monitor. - LOAD_ARCH, - JEQ32(Syscall::GetHostAuditArch(), JUMP(&l, past_arch_check_l)), - JEQ32(AUDIT_ARCH_X86_64, TRACE(Syscall::kX86_64)), - JEQ32(AUDIT_ARCH_I386, TRACE(Syscall::kX86_32)), - JEQ32(AUDIT_ARCH_PPC64LE, TRACE(Syscall::kPPC_64)), - TRACE(Syscall::kUnknown), - LABEL(&l, past_arch_check_l), + // If compiled arch is different than the runtime one, inform the Monitor. + LOAD_ARCH, + JEQ32(Syscall::GetHostAuditArch(), JUMP(&l, past_arch_check_l)), +#if defined(SAPI_X86_64) + JEQ32(AUDIT_ARCH_I386, TRACE(cpu::kX86)), // 32-bit sandboxee +#endif + TRACE(cpu::kUnknown), + LABEL(&l, past_arch_check_l), - // After the policy is uploaded, forkserver will execve the sandboxee. We - // need to allow this execve but not others. Since BPF does not have - // state, we need to inform the Monitor to decide, and for that we use a - // magic value in syscall args 5. Note that this value is not supposed to - // be secret, but just an optimization so that the monitor is not - // triggered on every call to execveat. - LOAD_SYSCALL_NR, - JNE32(__NR_execveat, JUMP(&l, past_execveat_l)), - ARG_32(4), - JNE32(AT_EMPTY_PATH, JUMP(&l, past_execveat_l)), - ARG_32(5), - JNE32(internal::kExecveMagic, JUMP(&l, past_execveat_l)), - SANDBOX2_TRACE, - LABEL(&l, past_execveat_l), + // After the policy is uploaded, forkserver will execve the sandboxee. We + // need to allow this execve but not others. Since BPF does not have + // state, we need to inform the Monitor to decide, and for that we use a + // magic value in syscall args 5. Note that this value is not supposed to + // be secret, but just an optimization so that the monitor is not + // triggered on every call to execveat. + LOAD_SYSCALL_NR, + JNE32(__NR_execveat, JUMP(&l, past_execveat_l)), + ARG_32(4), + JNE32(AT_EMPTY_PATH, JUMP(&l, past_execveat_l)), + ARG_32(5), + JNE32(internal::kExecveMagic, JUMP(&l, past_execveat_l)), + SANDBOX2_TRACE, + LABEL(&l, past_execveat_l), - // Forbid some syscalls because unsafe or too risky. - LOAD_SYSCALL_NR, - JEQ32(__NR_ptrace, DENY), - JEQ32(__NR_bpf, DENY), + // Forbid some syscalls because unsafe or too risky. + LOAD_SYSCALL_NR, + JEQ32(__NR_ptrace, DENY), + JEQ32(__NR_bpf, DENY), - // Disallow clone with CLONE_UNTRACED flag. - JNE32(__NR_clone, JUMP(&l, past_clone_untraced_l)), - // Regardless of arch, we only care about the lower 32-bits of the flags. - ARG_32(0), - JA32(CLONE_UNTRACED, DENY), - LABEL(&l, past_clone_untraced_l), + // Disallow clone with CLONE_UNTRACED flag. + JNE32(__NR_clone, JUMP(&l, past_clone_untraced_l)), + // Regardless of arch, we only care about the lower 32-bits of the flags. + ARG_32(0), + JA32(CLONE_UNTRACED, DENY), + LABEL(&l, past_clone_untraced_l), }; if (bpf_resolve_jumps(&l, policy.data(), policy.size()) != 0) { @@ -129,11 +129,16 @@ std::vector Policy::GetDefaultPolicy() const { std::vector Policy::GetTrackingPolicy() const { return { - LOAD_ARCH, - JEQ32(AUDIT_ARCH_X86_64, TRACE(Syscall::kX86_64)), - JEQ32(AUDIT_ARCH_I386, TRACE(Syscall::kX86_32)), - JEQ32(AUDIT_ARCH_PPC64LE, TRACE(Syscall::kPPC_64)), - TRACE(Syscall::kUnknown), + LOAD_ARCH, +#if defined(SAPI_X86_64) + JEQ32(AUDIT_ARCH_X86_64, TRACE(cpu::kX8664)), + JEQ32(AUDIT_ARCH_I386, TRACE(cpu::kX86)), +#elif defined(SAPI_PPC64_LE) + JEQ32(AUDIT_ARCH_PPC64LE, TRACE(cpu::kPPC64LE)), +#elif defined(SAPI_ARM64) + JEQ32(AUDIT_ARCH_AARCH64, TRACE(cpu::kArm64)), +#endif + TRACE(cpu::kUnknown), }; } diff --git a/sandboxed_api/sandbox2/policy_test.cc b/sandboxed_api/sandbox2/policy_test.cc index 809ecd9..a7cf852 100644 --- a/sandboxed_api/sandbox2/policy_test.cc +++ b/sandboxed_api/sandbox2/policy_test.cc @@ -25,6 +25,7 @@ #include "gtest/gtest.h" #include "absl/memory/memory.h" #include "absl/strings/string_view.h" +#include "sandboxed_api/sandbox2/config.h" #include "sandboxed_api/sandbox2/executor.h" #include "sandboxed_api/sandbox2/limits.h" #include "sandboxed_api/sandbox2/policybuilder.h" @@ -49,14 +50,21 @@ std::unique_ptr PolicyTestcasePolicy() { .AllowSyscall(__NR_close) .AllowSyscall(__NR_getppid) .AllowTCGETS() +#ifdef __NR_open .BlockSyscallWithErrno(__NR_open, ENOENT) +#endif .BlockSyscallWithErrno(__NR_openat, ENOENT) +#ifdef __NR_access .BlockSyscallWithErrno(__NR_access, ENOENT) +#endif +#ifdef __NR_faccessat + .BlockSyscallWithErrno(__NR_faccessat, ENOENT) +#endif .BlockSyscallWithErrno(__NR_prlimit64, EPERM) .BuildOrDie(); } -#if defined(__x86_64__) +#ifdef SAPI_X86_64 // Test that 32-bit syscalls from 64-bit are disallowed. TEST(PolicyTest, AMD64Syscall32PolicyAllowed) { SKIP_SANITIZERS_AND_COVERAGE; @@ -72,7 +80,7 @@ TEST(PolicyTest, AMD64Syscall32PolicyAllowed) { ASSERT_THAT(result.final_status(), Eq(Result::VIOLATION)); EXPECT_THAT(result.reason_code(), Eq(1)); // __NR_exit in 32-bit - EXPECT_THAT(result.GetSyscallArch(), Eq(Syscall::kX86_32)); + EXPECT_THAT(result.GetSyscallArch(), Eq(cpu::kX86)); } // Test that 32-bit syscalls from 64-bit for FS checks are disallowed. @@ -90,9 +98,9 @@ TEST(PolicyTest, AMD64Syscall32FsAllowed) { ASSERT_THAT(result.final_status(), Eq(Result::VIOLATION)); EXPECT_THAT(result.reason_code(), Eq(33)); // __NR_access in 32-bit - EXPECT_THAT(result.GetSyscallArch(), Eq(Syscall::kX86_32)); + EXPECT_THAT(result.GetSyscallArch(), Eq(cpu::kX86)); } -#endif // defined(__x86_64__) +#endif // Test that ptrace(2) is disallowed. TEST(PolicyTest, PtraceDisallowed) { @@ -161,7 +169,9 @@ std::unique_ptr MinimalTestcasePolicy() { .AllowStaticStartup() .AllowExit() .BlockSyscallWithErrno(__NR_prlimit64, EPERM) +#ifdef __NR_access .BlockSyscallWithErrno(__NR_access, ENOENT) +#endif .BuildOrDie(); } @@ -196,8 +206,10 @@ TEST(MinimalTest, MinimalSharedBinaryWorks) { .AllowOpen() .AllowExit() .AllowMmap() +#ifdef __NR_access // New glibc accesses /etc/ld.so.preload .BlockSyscallWithErrno(__NR_access, ENOENT) +#endif .BlockSyscallWithErrno(__NR_prlimit64, EPERM) .AddLibrariesForBinary(path) .BuildOrDie(); @@ -222,7 +234,9 @@ TEST(MallocTest, SystemMallocWorks) { .AllowSystemMalloc() .AllowExit() .BlockSyscallWithErrno(__NR_prlimit64, EPERM) +#ifdef __NR_access .BlockSyscallWithErrno(__NR_access, ENOENT) +#endif .BuildOrDie(); Sandbox2 s2(std::move(executor), std::move(policy)); @@ -246,7 +260,9 @@ TEST(MultipleSyscalls, AddPolicyOnSyscallsWorks) { auto policy = PolicyBuilder() +#ifdef __NR_open .BlockSyscallWithErrno(__NR_open, ENOENT) +#endif .BlockSyscallWithErrno(__NR_openat, ENOENT) .AllowStaticStartup() .AllowTcMalloc() @@ -257,7 +273,9 @@ TEST(MultipleSyscalls, AddPolicyOnSyscallsWorks) { .AddPolicyOnSyscalls({__NR_read, __NR_write}, {ERRNO(43)}) .AddPolicyOnSyscall(__NR_umask, {DENY}) .BlockSyscallWithErrno(__NR_prlimit64, EPERM) +#ifdef __NR_access .BlockSyscallWithErrno(__NR_access, ENOENT) +#endif .BuildOrDie(); Sandbox2 s2(std::move(executor), std::move(policy)); diff --git a/sandboxed_api/sandbox2/policybuilder.cc b/sandboxed_api/sandbox2/policybuilder.cc index 9eec379..678677d 100644 --- a/sandboxed_api/sandbox2/policybuilder.cc +++ b/sandboxed_api/sandbox2/policybuilder.cc @@ -15,13 +15,7 @@ #include "sandboxed_api/sandbox2/policybuilder.h" #include // For TCGETS -#if defined(__x86_64__) -#include -#endif -#if defined(__powerpc64__) -#include // On PPC, TCGETS macro needs termios -#endif -#include // For the fcntl flags +#include // For the fcntl flags #include #include // For SYS_CONNECT #include // For GRND_NONBLOCK @@ -34,13 +28,21 @@ #include #include "absl/memory/memory.h" +#include "absl/status/statusor.h" #include "absl/strings/escaping.h" #include "absl/strings/match.h" +#include "sandboxed_api/sandbox2/config.h" #include "sandboxed_api/sandbox2/namespace.h" #include "sandboxed_api/sandbox2/util/bpf_helper.h" #include "sandboxed_api/sandbox2/util/path.h" #include "sandboxed_api/util/status_macros.h" +#if defined(SAPI_X86_64) +#include +#elif defined(SAPI_PPC64_LE) +#include // On PPC, TCGETS macro needs termios +#endif + namespace sandbox2 { namespace { @@ -510,7 +512,7 @@ PolicyBuilder& PolicyBuilder::AllowStaticStartup() { JEQ32(SIG_UNBLOCK, ALLOW), }); -#if defined(__x86_64__) +#ifdef SAPI_X86_64 // The second argument is a pointer. AddPolicyOnSyscall(__NR_arch_prctl, { ARG_32(0), @@ -518,7 +520,12 @@ PolicyBuilder& PolicyBuilder::AllowStaticStartup() { }); #endif + if constexpr (host_cpu::IsArm64()) { + BlockSyscallWithErrno(__NR_readlinkat, ENOENT); + } +#ifdef __NR_readlink BlockSyscallWithErrno(__NR_readlink, ENOENT); +#endif return *this; } @@ -526,7 +533,11 @@ PolicyBuilder& PolicyBuilder::AllowStaticStartup() { PolicyBuilder& PolicyBuilder::AllowDynamicStartup() { AllowRead(); AllowStat(); - AllowSyscalls({__NR_lseek, __NR_close, __NR_munmap}); + AllowSyscalls({__NR_lseek, +#ifdef __NR__llseek + __NR__llseek, // Newer glibc on PPC +#endif + __NR_close, __NR_munmap}); AddPolicyOnSyscall(__NR_mprotect, { ARG_32(2), JEQ32(PROT_READ, ALLOW), @@ -656,7 +667,7 @@ PolicyBuilder& PolicyBuilder::DangerDefaultAllowAll() { return *this; } -sapi::StatusOr PolicyBuilder::ValidateAbsolutePath( +absl::StatusOr PolicyBuilder::ValidateAbsolutePath( absl::string_view path) { if (!file::IsAbsolutePath(path)) { return absl::InvalidArgumentError( @@ -665,7 +676,7 @@ sapi::StatusOr PolicyBuilder::ValidateAbsolutePath( return ValidatePath(path); } -sapi::StatusOr PolicyBuilder::ValidatePath( +absl::StatusOr PolicyBuilder::ValidatePath( absl::string_view path) { std::string fixed_path = file::CleanPath(path); if (fixed_path != path) { @@ -686,7 +697,7 @@ std::vector PolicyBuilder::ResolveBpfFunc(BpfFunc f) { return policy; } -sapi::StatusOr> PolicyBuilder::TryBuild() { +absl::StatusOr> PolicyBuilder::TryBuild() { auto output = absl::WrapUnique(new Policy()); if (!last_status_.ok()) { @@ -879,7 +890,9 @@ PolicyBuilder& PolicyBuilder::AddNetworkProxyPolicy() { AllowFutexOp(FUTEX_WAIT); AllowFutexOp(FUTEX_WAIT_BITSET); AllowSyscalls({ +#ifdef __NR_dup2 __NR_dup2, +#endif __NR_recvmsg, __NR_close, __NR_gettid, @@ -899,7 +912,7 @@ PolicyBuilder& PolicyBuilder::AddNetworkProxyPolicy() { LABEL(&labels, getsockopt_end), }; }); -#if defined(__powerpc64__) +#ifdef SAPI_PPC64_LE AddPolicyOnSyscall(__NR_socketcall, { ARG_32(0), JEQ32(SYS_SOCKET, ALLOW), @@ -925,7 +938,7 @@ PolicyBuilder& PolicyBuilder::AddNetworkProxyHandlerPolicy() { }); AddPolicyOnSyscall(__NR_connect, {TRAP(0)}); -#if defined(__powerpc64__) +#ifdef SAPI_PPC64_LE AddPolicyOnSyscall(__NR_socketcall, { ARG_32(0), JEQ32(SYS_CONNECT, TRAP(0)), diff --git a/sandboxed_api/sandbox2/policybuilder.h b/sandboxed_api/sandbox2/policybuilder.h index f05023c..5a5fd97 100644 --- a/sandboxed_api/sandbox2/policybuilder.h +++ b/sandboxed_api/sandbox2/policybuilder.h @@ -29,18 +29,16 @@ #include #include "absl/base/macros.h" #include "absl/memory/memory.h" +#include "absl/status/statusor.h" #include "absl/strings/string_view.h" #include "sandboxed_api/sandbox2/mounts.h" #include "sandboxed_api/sandbox2/network_proxy/filtering.h" #include "sandboxed_api/sandbox2/policy.h" -#include "sandboxed_api/util/statusor.h" struct bpf_labels; namespace sandbox2 { -constexpr char kDefaultHostname[] = "sandbox2"; - // PolicyBuilder is a helper class to simplify creation of policies. The builder // uses fluent interface for convenience and increased readability of policies. // @@ -91,6 +89,8 @@ constexpr char kDefaultHostname[] = "sandbox2"; // For a more complicated example, see examples/persistent/persistent_sandbox.cc class PolicyBuilder final { public: + static constexpr absl::string_view kDefaultHostname = "sandbox2"; + using BpfInitializer = std::initializer_list; using BpfFunc = const std::function(bpf_labels&)>&; using SyscallInitializer = std::initializer_list; @@ -390,7 +390,7 @@ class PolicyBuilder final { // Builds the policy returning a unique_ptr to it. This should only be called // once. - sapi::StatusOr> TryBuild(); + absl::StatusOr> TryBuild(); // Builds the policy returning a unique_ptr to it. This should only be called // once. @@ -532,9 +532,9 @@ class PolicyBuilder final { std::vector ResolveBpfFunc(BpfFunc f); - static sapi::StatusOr ValidateAbsolutePath( + static absl::StatusOr ValidateAbsolutePath( absl::string_view path); - static sapi::StatusOr ValidatePath(absl::string_view path); + static absl::StatusOr ValidatePath(absl::string_view path); void StoreDescription(PolicyBuilderDescription* pb_description); @@ -542,7 +542,7 @@ class PolicyBuilder final { bool use_namespaces_ = true; bool requires_namespaces_ = false; bool allow_unrestricted_networking_ = false; - std::string hostname_ = kDefaultHostname; + std::string hostname_ = std::string(kDefaultHostname); bool collect_stacktrace_on_violation_ = true; bool collect_stacktrace_on_signal_ = true; diff --git a/sandboxed_api/sandbox2/policybuilder_test.cc b/sandboxed_api/sandbox2/policybuilder_test.cc index a86befd..652a827 100644 --- a/sandboxed_api/sandbox2/policybuilder_test.cc +++ b/sandboxed_api/sandbox2/policybuilder_test.cc @@ -24,6 +24,7 @@ #include "gmock/gmock.h" #include "gtest/gtest.h" #include "absl/memory/memory.h" +#include "absl/status/statusor.h" #include "absl/strings/match.h" #include "absl/strings/str_cat.h" #include "absl/strings/str_split.h" @@ -57,7 +58,7 @@ class PolicyBuilderPeer { int policy_size() const { return builder_->user_policy_.size(); } - static sapi::StatusOr ValidateAbsolutePath( + static absl::StatusOr ValidateAbsolutePath( absl::string_view path) { return PolicyBuilder::ValidateAbsolutePath(path); } @@ -100,7 +101,7 @@ TEST_F(PolicyBuilderTest, Testpolicy_size) { builder.AllowSystemMalloc(); assert_increased(); builder.AllowSyscall(__NR_munmap); assert_same(); builder.BlockSyscallWithErrno(__NR_munmap, 1); assert_same(); - builder.BlockSyscallWithErrno(__NR_open, 1); + builder.BlockSyscallWithErrno(__NR_openat, 1); assert_increased(); builder.AllowTCGETS(); assert_increased(); diff --git a/sandboxed_api/sandbox2/regs.cc b/sandboxed_api/sandbox2/regs.cc index 6dfcd14..7a4c214 100644 --- a/sandboxed_api/sandbox2/regs.cc +++ b/sandboxed_api/sandbox2/regs.cc @@ -23,65 +23,108 @@ #include +#include "absl/base/macros.h" +#include "absl/status/status.h" #include "absl/strings/str_cat.h" +#include "sandboxed_api/sandbox2/config.h" #include "sandboxed_api/sandbox2/util/strerror.h" namespace sandbox2 { -absl::Status Regs::Fetch() { -#if defined(__powerpc64__) - iovec pt_iov = {&user_regs_, sizeof(user_regs_)}; +#ifndef NT_ARM_SYSTEM_CALL +#define NT_ARM_SYSTEM_CALL 0x404 +#endif - if (ptrace(PTRACE_GETREGSET, pid_, NT_PRSTATUS, &pt_iov) == -1L) { - return absl::InternalError(absl::StrCat( - "ptrace(PTRACE_GETREGSET, pid=", pid_, ") failed: ", StrError(errno))); - } - if (pt_iov.iov_len != sizeof(user_regs_)) { - return absl::InternalError(absl::StrCat( - "ptrace(PTRACE_GETREGSET, pid=", pid_, - ") size returned: ", pt_iov.iov_len, - " different than sizeof(user_regs_): ", sizeof(user_regs_))); - } -#else +absl::Status Regs::Fetch() { +#ifdef SAPI_X86_64 if (ptrace(PTRACE_GETREGS, pid_, 0, &user_regs_) == -1L) { return absl::InternalError(absl::StrCat("ptrace(PTRACE_GETREGS, pid=", pid_, ") failed: ", StrError(errno))); } #endif + if constexpr (host_cpu::IsPPC64LE() || host_cpu::IsArm64()) { + iovec pt_iov = {&user_regs_, sizeof(user_regs_)}; + + if (ptrace(PTRACE_GETREGSET, pid_, NT_PRSTATUS, &pt_iov) == -1L) { + return absl::InternalError( + absl::StrCat("ptrace(PTRACE_GETREGSET, pid=", pid_, + ") failed: ", StrError(errno))); + } + if (pt_iov.iov_len != sizeof(user_regs_)) { + return absl::InternalError(absl::StrCat( + "ptrace(PTRACE_GETREGSET, pid=", pid_, + ") size returned: ", pt_iov.iov_len, + " different than sizeof(user_regs_): ", sizeof(user_regs_))); + } + + // On AArch64, we are not done yet. Read the syscall number. + if constexpr (host_cpu::IsArm64()) { + iovec sys_iov = {&syscall_number_, sizeof(syscall_number_)}; + + if (ptrace(PTRACE_GETREGSET, pid_, NT_ARM_SYSTEM_CALL, &sys_iov) == -1L) { + return absl::InternalError( + absl::StrCat("ptrace(PTRACE_GETREGSET, pid=", pid_, + ", NT_ARM_SYSTEM_CALL) failed: ", StrError(errno))); + } + if (sys_iov.iov_len != sizeof(syscall_number_)) { + return absl::InternalError(absl::StrCat( + "ptrace(PTRACE_GETREGSET, pid=", pid_, + ", NT_ARM_SYSTEM_CALL) size returned: ", sys_iov.iov_len, + " different than sizeof(syscall_number_): ", + sizeof(syscall_number_))); + } + } + } return absl::OkStatus(); } absl::Status Regs::Store() { -#if defined(__powerpc64__) - iovec pt_iov = {&user_regs_, sizeof(user_regs_)}; - - if (ptrace(PTRACE_SETREGSET, pid_, NT_PRSTATUS, &pt_iov) == -1L) { - return absl::InternalError(absl::StrCat( - "ptrace(PTRACE_SETREGSET, pid=", pid_, ") failed: ", StrError(errno))); - } -#else +#ifdef SAPI_X86_64 if (ptrace(PTRACE_SETREGS, pid_, 0, &user_regs_) == -1) { return absl::InternalError(absl::StrCat("ptrace(PTRACE_SETREGS, pid=", pid_, ") failed: ", StrError(errno))); } #endif + if constexpr (host_cpu::IsPPC64LE() || host_cpu::IsArm64()) { + iovec pt_iov = {&user_regs_, sizeof(user_regs_)}; + + if (ptrace(PTRACE_SETREGSET, pid_, NT_PRSTATUS, &pt_iov) == -1L) { + return absl::InternalError( + absl::StrCat("ptrace(PTRACE_SETREGSET, pid=", pid_, + ") failed: ", StrError(errno))); + } + + // Store syscall number on AArch64. + if constexpr (host_cpu::IsArm64()) { + iovec sys_iov = {&syscall_number_, sizeof(syscall_number_)}; + + if (ptrace(PTRACE_SETREGSET, pid_, NT_ARM_SYSTEM_CALL, &sys_iov) == -1L) { + return absl::InternalError( + absl::StrCat("ptrace(PTRACE_SETREGSET, pid=", pid_, + ", NT_ARM_SYSTEM_CALL) failed: ", StrError(errno))); + } + } + } return absl::OkStatus(); } absl::Status Regs::SkipSyscallReturnValue(uint64_t value) { -#if defined(__x86_64__) +#if defined(SAPI_X86_64) user_regs_.orig_rax = -1; user_regs_.rax = value; -#elif defined(__powerpc64__) +#elif defined(SAPI_PPC64_LE) user_regs_.gpr[0] = -1; user_regs_.gpr[3] = value; +#elif defined(SAPI_ARM64) + user_regs_.regs[0] = -1; + syscall_number_ = value; #endif return Store(); } -Syscall Regs::ToSyscall(Syscall::CpuArch syscall_arch) const { -#if defined(__x86_64__) - if (ABSL_PREDICT_TRUE(syscall_arch == Syscall::kX86_64)) { +Syscall Regs::ToSyscall(cpu::Architecture syscall_arch) const { +#if defined(SAPI_X86_64) + if (ABSL_PREDICT_TRUE(syscall_arch == cpu::kX8664)) { auto syscall = user_regs_.orig_rax; Syscall::Args args = {user_regs_.rdi, user_regs_.rsi, user_regs_.rdx, user_regs_.r10, user_regs_.r8, user_regs_.r9}; @@ -89,7 +132,7 @@ Syscall Regs::ToSyscall(Syscall::CpuArch syscall_arch) const { auto ip = user_regs_.rip; return Syscall(syscall_arch, syscall, args, pid_, sp, ip); } - if (syscall_arch == Syscall::kX86_32) { + if (syscall_arch == cpu::kX86) { auto syscall = user_regs_.orig_rax & 0xFFFFFFFF; Syscall::Args args = { user_regs_.rbx & 0xFFFFFFFF, user_regs_.rcx & 0xFFFFFFFF, @@ -99,8 +142,8 @@ Syscall Regs::ToSyscall(Syscall::CpuArch syscall_arch) const { auto ip = user_regs_.rip & 0xFFFFFFFF; return Syscall(syscall_arch, syscall, args, pid_, sp, ip); } -#elif defined(__powerpc64__) - if (ABSL_PREDICT_TRUE(syscall_arch == Syscall::kPPC_64)) { +#elif defined(SAPI_PPC64_LE) + if (ABSL_PREDICT_TRUE(syscall_arch == cpu::kPPC64LE)) { auto syscall = user_regs_.gpr[0]; Syscall::Args args = {user_regs_.orig_gpr3, user_regs_.gpr[4], user_regs_.gpr[5], user_regs_.gpr[6], @@ -109,12 +152,28 @@ Syscall Regs::ToSyscall(Syscall::CpuArch syscall_arch) const { auto ip = user_regs_.nip; return Syscall(syscall_arch, syscall, args, pid_, sp, ip); } +#elif defined(SAPI_ARM64) + if (ABSL_PREDICT_TRUE(syscall_arch == cpu::kArm64)) { + Syscall::Args args = { + // First argument should be orig_x0, which is not available to ptrace on + // AArch64 (see + // https://undo.io/resources/arm64-vs-arm32-whats-different-linux-programmers/), + // as it will have been overwritten. For our use case, though, using + // regs[0] is fine, as we are always called on syscall entry and never + // on exit. + user_regs_.regs[0], user_regs_.regs[1], user_regs_.regs[2], + user_regs_.regs[3], user_regs_.regs[4], user_regs_.regs[5], + }; + auto sp = user_regs_.sp; + auto ip = user_regs_.pc; + return Syscall(syscall_arch, syscall_number_, args, pid_, sp, ip); + } #endif return Syscall(pid_); } void Regs::StoreRegisterValuesInProtobuf(RegisterValues* values) const { -#if defined(__x86_64__) +#if defined(SAPI_X86_64) RegisterX8664* regs = values->mutable_register_x86_64(); regs->set_r15(user_regs_.r15); regs->set_r14(user_regs_.r14); @@ -143,7 +202,7 @@ void Regs::StoreRegisterValuesInProtobuf(RegisterValues* values) const { regs->set_es(user_regs_.es); regs->set_fs(user_regs_.fs); regs->set_gs(user_regs_.gs); -#elif defined(__powerpc64__) +#elif defined(SAPI_PPC64_LE) RegisterPowerpc64* regs = values->mutable_register_powerpc64(); for (int i = 0; i < ABSL_ARRAYSIZE(user_regs_.gpr); ++i) { regs->add_gpr(user_regs_.gpr[i]); @@ -164,6 +223,14 @@ void Regs::StoreRegisterValuesInProtobuf(RegisterValues* values) const { regs->set_zero1(user_regs_.zero1); regs->set_zero2(user_regs_.zero2); regs->set_zero3(user_regs_.zero3); +#elif defined(SAPI_ARM64) + RegisterAarch64* regs = values->mutable_register_aarch64(); + for (int i = 0; i < ABSL_ARRAYSIZE(user_regs_.regs); ++i) { + regs->add_regs(user_regs_.regs[i]); + } + regs->set_sp(user_regs_.sp); + regs->set_pc(user_regs_.pc); + regs->set_pstate(user_regs_.pstate); #endif } diff --git a/sandboxed_api/sandbox2/regs.h b/sandboxed_api/sandbox2/regs.h index f1149a9..cce7022 100644 --- a/sandboxed_api/sandbox2/regs.h +++ b/sandboxed_api/sandbox2/regs.h @@ -24,6 +24,7 @@ #include #include "absl/status/status.h" +#include "sandboxed_api/sandbox2/config.h" #include "sandboxed_api/sandbox2/syscall.h" #include "sandboxed_api/sandbox2/violation.pb.h" @@ -33,10 +34,6 @@ namespace sandbox2 { // assumes the process is already attached. class Regs { public: -#if !defined(__x86_64__) && !defined(__powerpc64__) - static_assert(false, "No support for the current CPU architecture"); -#endif - explicit Regs(pid_t pid) : pid_(pid) {} // Copies register values from the process @@ -49,7 +46,7 @@ class Regs { absl::Status SkipSyscallReturnValue(uint64_t value); // Converts raw register values obtained on syscall entry to syscall info - Syscall ToSyscall(Syscall::CpuArch syscall_arch) const; + Syscall ToSyscall(cpu::Architecture syscall_arch) const; pid_t pid() const { return pid_; } @@ -60,7 +57,7 @@ class Regs { friend class StackTracePeer; struct PtraceRegisters { -#if defined(__x86_64__) +#if defined(SAPI_X86_64) uint64_t r15; uint64_t r14; uint64_t r13; @@ -88,7 +85,7 @@ class Regs { uint64_t es; uint64_t fs; uint64_t gs; -#elif defined(__powerpc64__) +#elif defined(SAPI_PPC64_LE) uint64_t gpr[32]; uint64_t nip; uint64_t msr; @@ -108,6 +105,13 @@ class Regs { uint64_t zero1; uint64_t zero2; uint64_t zero3; +#elif defined(SAPI_ARM64) + uint64_t regs[31]; + uint64_t sp; + uint64_t pc; + uint64_t pstate; +#else + static_assert(false, "Host CPU architecture not supported, see config.h"); #endif }; @@ -116,6 +120,9 @@ class Regs { // Registers fetched with ptrace(PR_GETREGS/GETREGSET, pid). PtraceRegisters user_regs_ = {}; + + // On AArch64, obtaining the syscall number needs a specific call to ptrace() + int syscall_number_ = 0; }; } // namespace sandbox2 diff --git a/sandboxed_api/sandbox2/result.h b/sandboxed_api/sandbox2/result.h index f381ccb..d2d9c5a 100644 --- a/sandboxed_api/sandbox2/result.h +++ b/sandboxed_api/sandbox2/result.h @@ -28,6 +28,7 @@ #include "absl/memory/memory.h" #include "absl/status/status.h" +#include "sandboxed_api/sandbox2/config.h" #include "sandboxed_api/sandbox2/regs.h" #include "sandboxed_api/sandbox2/syscall.h" @@ -131,8 +132,8 @@ class Result { // Returns the current syscall architecture. // Client architecture when final_status_ == VIOLATION, might be different // from the host architecture (32-bit vs 64-bit syscalls). - Syscall::CpuArch GetSyscallArch() const { - return syscall_ ? syscall_->arch() : Syscall::kUnknown; + cpu::Architecture GetSyscallArch() const { + return syscall_ ? syscall_->arch() : cpu::kUnknown; } const std::vector stack_trace() { return stack_trace_; } diff --git a/sandboxed_api/sandbox2/sandbox2.cc b/sandboxed_api/sandbox2/sandbox2.cc index 559bceb..d0c1e22 100644 --- a/sandboxed_api/sandbox2/sandbox2.cc +++ b/sandboxed_api/sandbox2/sandbox2.cc @@ -21,6 +21,7 @@ #include #include "absl/memory/memory.h" +#include "absl/status/statusor.h" #include "absl/time/time.h" #include "sandboxed_api/sandbox2/monitor.h" #include "sandboxed_api/sandbox2/result.h" @@ -33,7 +34,7 @@ Sandbox2::~Sandbox2() { } } -sapi::StatusOr Sandbox2::AwaitResultWithTimeout( +absl::StatusOr Sandbox2::AwaitResultWithTimeout( absl::Duration timeout) { CHECK(monitor_ != nullptr) << "Sandbox was not launched yet"; CHECK(monitor_thread_ != nullptr) << "Sandbox was already waited on"; diff --git a/sandboxed_api/sandbox2/sandbox2.h b/sandboxed_api/sandbox2/sandbox2.h index 9527ff2..62dd4fe 100644 --- a/sandboxed_api/sandbox2/sandbox2.h +++ b/sandboxed_api/sandbox2/sandbox2.h @@ -26,6 +26,7 @@ #include #include "absl/base/macros.h" #include "absl/memory/memory.h" +#include "absl/status/statusor.h" #include "sandboxed_api/sandbox2/comms.h" #include "sandboxed_api/sandbox2/executor.h" #include "sandboxed_api/sandbox2/ipc.h" @@ -33,7 +34,6 @@ #include "sandboxed_api/sandbox2/notify.h" #include "sandboxed_api/sandbox2/policy.h" #include "sandboxed_api/sandbox2/result.h" -#include "sandboxed_api/util/statusor.h" namespace sandbox2 { @@ -76,7 +76,7 @@ class Sandbox2 final { // Waits for sandbox execution to finish within the timeout. // Returns execution result or a DeadlineExceededError if the sandboxee does // not finish in time. - sapi::StatusOr AwaitResultWithTimeout(absl::Duration timeout); + absl::StatusOr AwaitResultWithTimeout(absl::Duration timeout); // Requests termination of the sandboxee. // Sandbox should still waited with AwaitResult(), as it may finish for other diff --git a/sandboxed_api/sandbox2/sandbox2_test.cc b/sandboxed_api/sandbox2/sandbox2_test.cc index edf3d6c..df94c1f 100644 --- a/sandboxed_api/sandbox2/sandbox2_test.cc +++ b/sandboxed_api/sandbox2/sandbox2_test.cc @@ -26,6 +26,7 @@ #include "gtest/gtest.h" #include "absl/memory/memory.h" #include "absl/strings/str_cat.h" +#include "sandboxed_api/sandbox2/config.h" #include "sandboxed_api/sandbox2/executor.h" #include "sandboxed_api/sandbox2/policy.h" #include "sandboxed_api/sandbox2/policybuilder.h" diff --git a/sandboxed_api/sandbox2/stack_trace.cc b/sandboxed_api/sandbox2/stack_trace.cc index 7df7952..ef494b1 100644 --- a/sandboxed_api/sandbox2/stack_trace.cc +++ b/sandboxed_api/sandbox2/stack_trace.cc @@ -31,6 +31,7 @@ #include "absl/strings/strip.h" #include "libcap/include/sys/capability.h" #include "sandboxed_api/sandbox2/comms.h" +#include "sandboxed_api/sandbox2/config.h" #include "sandboxed_api/sandbox2/executor.h" #include "sandboxed_api/sandbox2/ipc.h" #include "sandboxed_api/sandbox2/limits.h" @@ -85,6 +86,9 @@ std::unique_ptr StackTracePeer::GetPolicy(pid_t target_pid, // libunwind .AllowSyscall(__NR_fstat) .AllowSyscall(__NR_lseek) +#ifdef __NR__llseek + .AllowSyscall(__NR__llseek) // Newer glibc on PPC +#endif .AllowSyscall(__NR_mincore) .AllowSyscall(__NR_mprotect) .AllowSyscall(__NR_munmap) @@ -270,6 +274,9 @@ bool StackTracePeer::LaunchLibunwindSandbox(const Regs* regs, } std::vector GetStackTrace(const Regs* regs, const Mounts& mounts) { + if constexpr (host_cpu::IsArm64()) { + return {"[Stack traces unavailable]"}; + } if (absl::GetFlag(FLAGS_sandbox_disable_all_stack_traces)) { return {"[Stacktraces disabled]"}; } diff --git a/sandboxed_api/sandbox2/syscall.cc b/sandboxed_api/sandbox2/syscall.cc index 2b3ffad..980320a 100644 --- a/sandboxed_api/sandbox2/syscall.cc +++ b/sandboxed_api/sandbox2/syscall.cc @@ -12,12 +12,11 @@ // See the License for the specific language governing permissions and // limitations under the License. -// Implementation of the sandbox2::Syscall class. - #include "sandboxed_api/sandbox2/syscall.h" #include #include + #include #include #include @@ -26,6 +25,7 @@ #include #include "absl/strings/str_format.h" #include "absl/strings/str_join.h" +#include "sandboxed_api/sandbox2/config.h" #include "sandboxed_api/sandbox2/syscall_defs.h" #ifndef AUDIT_ARCH_PPC64LE @@ -34,46 +34,43 @@ namespace sandbox2 { -std::string Syscall::GetArchDescription(CpuArch arch) { +std::string Syscall::GetArchDescription(cpu::Architecture arch) { switch (arch) { - case kX86_64: + case cpu::kX8664: return "[X86-64]"; - case kX86_32: + case cpu::kX86: return "[X86-32]"; - case kPPC_64: + case cpu::kPPC64LE: return "[PPC-64]"; + case cpu::kArm64: + return "[Arm-64]"; default: LOG(ERROR) << "Unknown CPU architecture: " << arch; return absl::StrFormat("[UNKNOWN_ARCH:%d]", arch); } } -Syscall::CpuArch Syscall::GetHostArch() { -#if defined(__x86_64__) - return kX86_64; -#elif defined(__i386__) - return kX86_32; -#elif defined(__powerpc64__) - return kPPC_64; -#endif -} - uint32_t Syscall::GetHostAuditArch() { -#if defined(__x86_64__) - return AUDIT_ARCH_X86_64; -#elif defined(__i386__) - return AUDIT_ARCH_I386; -#elif defined(__powerpc64__) - return AUDIT_ARCH_PPC64LE; -#endif + switch (host_cpu::Architecture()) { + case cpu::kX8664: + return AUDIT_ARCH_X86_64; + case cpu::kPPC64LE: + return AUDIT_ARCH_PPC64LE; + case cpu::kArm64: + return AUDIT_ARCH_AARCH64; + default: + // The static_assert() in config.h should prevent us from ever getting + // here. + return 0; // Not reached + } } std::string Syscall::GetName() const { - absl::string_view name = SyscallTable::get(arch_).GetName(nr_); - if (name.empty()) { - return absl::StrFormat("UNKNOWN[%d/0x%x]", nr_, nr_); + if (absl::string_view name = SyscallTable::get(arch_).GetName(nr_); + !name.empty()) { + return std::string(name); } - return std::string(name); + return absl::StrFormat("UNKNOWN[%d/0x%x]", nr_, nr_); } std::vector Syscall::GetArgumentsDescription() const { diff --git a/sandboxed_api/sandbox2/syscall.h b/sandboxed_api/sandbox2/syscall.h index 50f8dbf..132a505 100644 --- a/sandboxed_api/sandbox2/syscall.h +++ b/sandboxed_api/sandbox2/syscall.h @@ -13,7 +13,7 @@ // limitations under the License. // The sandbox2::Syscalls class defines mostly static helper methods which -// are used to analyze status of the ptraced process +// are used to analyze the status of the sandboxed process. #ifndef SANDBOXED_API_SANDBOX2_SYSCALL_H__ #define SANDBOXED_API_SANDBOX2_SYSCALL_H__ @@ -26,40 +26,34 @@ #include #include +#include "sandboxed_api/sandbox2/config.h" + namespace sandbox2 { class Syscall { public: - // Supported CPU architectures. - // Linux: Use a magic value, so it can be easily spotted in the seccomp-bpf - // bytecode decompilation stream. Must be < (1<<15), as/ that's the size of - // data which can be returned by BPF. - enum CpuArch { - kUnknown = 0xCAF0, - kX86_64, - kX86_32, - kPPC_64, - }; // Maximum number of syscall arguments static constexpr size_t kMaxArgs = 6; using Args = std::array; // Returns the host architecture, according to CpuArch. - static CpuArch GetHostArch(); + static constexpr cpu::Architecture GetHostArch() { + return host_cpu::Architecture(); + } // Returns the host architecture, according to . static uint32_t GetHostAuditArch(); // Returns a description of the architecture. - static std::string GetArchDescription(CpuArch arch); + static std::string GetArchDescription(cpu::Architecture arch); Syscall() = default; - Syscall(CpuArch arch, uint64_t nr, Args args = {}) + Syscall(cpu::Architecture arch, uint64_t nr, Args args = {}) : arch_(arch), nr_(nr), args_(args) {} pid_t pid() const { return pid_; } uint64_t nr() const { return nr_; } - CpuArch arch() const { return arch_; } + cpu::Architecture arch() const { return arch_; } const Args& args() const { return args_; } uint64_t stack_pointer() const { return sp_; } uint64_t instruction_pointer() const { return ip_; } @@ -72,12 +66,12 @@ class Syscall { private: friend class Regs; - Syscall(pid_t pid) : pid_(pid) {} - Syscall(CpuArch arch, uint64_t nr, Args args, pid_t pid, uint64_t sp, - uint64_t ip) + explicit Syscall(pid_t pid) : pid_(pid) {} + Syscall(cpu::Architecture arch, uint64_t nr, Args args, pid_t pid, + uint64_t sp, uint64_t ip) : arch_(arch), nr_(nr), args_(args), pid_(pid), sp_(sp), ip_(ip) {} - CpuArch arch_ = kUnknown; + cpu::Architecture arch_ = cpu::kUnknown; uint64_t nr_ = -1; Args args_ = {}; pid_t pid_ = -1; diff --git a/sandboxed_api/sandbox2/syscall_defs.cc b/sandboxed_api/sandbox2/syscall_defs.cc index 1a8c94f..b79f6c7 100644 --- a/sandboxed_api/sandbox2/syscall_defs.cc +++ b/sandboxed_api/sandbox2/syscall_defs.cc @@ -7,6 +7,7 @@ #include "absl/strings/escaping.h" #include "absl/strings/str_cat.h" #include "absl/strings/str_format.h" +#include "sandboxed_api/sandbox2/config.h" #include "sandboxed_api/sandbox2/util.h" namespace sandbox2 { @@ -128,7 +129,6 @@ std::vector SyscallTable::GetArgumentsDescription( #define SYSCALLS_UNUSED00_99(prefix) \ SYSCALLS_UNUSED00_49(prefix), SYSCALLS_UNUSED50_99(prefix) -#if defined(__x86_64__) // Syscall description table for Linux x86_64 constexpr SyscallTable::Entry kSyscallDataX8664[] = { MakeEntry("read", kInt, kHex, kInt), // 0 @@ -824,12 +824,10 @@ constexpr SyscallTable::Entry kSyscallDataX8632[] = { MakeEntry("bpf", kHex, kHex, kHex, kHex, kHex, kHex), // 357 }; -#elif defined(__powerpc64__) - // http://lxr.free-electrons.com/source/arch/powerpc/include/uapi/asm/unistd.h // Note: PPC64 syscalls can have up to 7 register arguments, but nobody is // using the 7th argument - probably for x64 compatibility reasons. -constexpr SyscallTable::Entry kSyscallDataPPC64[] = { +constexpr SyscallTable::Entry kSyscallDataPPC64LE[] = { MakeEntry("restart_syscall", kGen, kGen, kGen, kGen, kGen, kGen), // 0 MakeEntry("exit", kInt, kGen, kGen, kGen, kGen, kGen), // 1 MakeEntry("fork", kGen, kGen, kGen, kGen, kGen, kGen), // 2 @@ -1218,7 +1216,290 @@ constexpr SyscallTable::Entry kSyscallDataPPC64[] = { MakeEntry("pwritev2", kHex, kHex, kHex, kHex, kHex, kHex), // 381 }; -#endif +// TODO(cblichmann): Confirm the entries in this list. +// https://github.com/torvalds/linux/blob/v5.8/include/uapi/asm-generic/unistd.h +constexpr SyscallTable::Entry kSyscallDataArm64[] = { + MakeEntry("io_setup", UnknownArguments()), // 0 + MakeEntry("io_destroy", UnknownArguments()), // 1 + MakeEntry("io_submit", UnknownArguments()), // 2 + MakeEntry("io_cancel", UnknownArguments()), // 3 + MakeEntry("io_getevents", UnknownArguments()), // 4 + MakeEntry("setxattr", kPath, kString, kGen, kInt, kHex, kGen), // 5 + MakeEntry("lsetxattr", kPath, kString, kGen, kInt, kHex, kGen), // 6 + MakeEntry("fsetxattr", UnknownArguments()), // 7 + MakeEntry("getxattr", kPath, kString, kGen, kInt, kGen, kGen), // 8 + MakeEntry("lgetxattr", kPath, kString, kGen, kInt, kGen, kGen), // 9 + MakeEntry("fgetxattr", UnknownArguments()), // 10 + MakeEntry("listxattr", kPath, kGen, kInt, kGen, kGen, kGen), // 11 + MakeEntry("llistxattr", kPath, kGen, kInt, kGen, kGen, kGen), // 12 + MakeEntry("flistxattr", UnknownArguments()), // 13 + MakeEntry("removexattr", kPath, kString, kGen, kGen, kGen, kGen), // 14 + MakeEntry("lremovexattr", UnknownArguments()), // 15 + MakeEntry("fremovexattr", UnknownArguments()), // 16 + MakeEntry("getcwd", UnknownArguments()), // 17 + MakeEntry("lookup_dcookie", UnknownArguments()), // 18 + MakeEntry("eventfd2", UnknownArguments()), // 19 + MakeEntry("epoll_create1", UnknownArguments()), // 20 + MakeEntry("epoll_ctl", UnknownArguments()), // 21 + MakeEntry("epoll_pwait", UnknownArguments()), // 22 + MakeEntry("dup", UnknownArguments()), // 23 + MakeEntry("dup3", UnknownArguments()), // 24 + MakeEntry("fcntl", UnknownArguments()), // 25 + MakeEntry("inotify_init1", UnknownArguments()), // 26 + MakeEntry("inotify_add_watch", UnknownArguments()), // 27 + MakeEntry("inotify_rm_watch", UnknownArguments()), // 28 + MakeEntry("ioctl", UnknownArguments()), // 29 + MakeEntry("ioprio_set", UnknownArguments()), // 30 + MakeEntry("ioprio_get", UnknownArguments()), // 31 + MakeEntry("flock", UnknownArguments()), // 32 + MakeEntry("mknodat", kGen, kPath, kGen, kGen, kGen, kGen), // 33 + MakeEntry("mkdirat", kGen, kPath, kGen, kGen, kGen, kGen), // 34 + MakeEntry("unlinkat", kGen, kPath, kGen, kGen, kGen, kGen), // 35 + MakeEntry("symlinkat", kPath, kGen, kPath, kGen, kGen, kGen), // 36 + MakeEntry("linkat", kGen, kPath, kGen, kPath, kGen, kGen), // 37 + MakeEntry("renameat", kGen, kPath, kGen, kPath, kGen, kGen), // 38 + MakeEntry("umount2", kPath, kHex, kGen, kGen, kGen, kGen), // 39 + MakeEntry("mount", kPath, kPath, kString, kHex, kGen, kGen), // 40 + MakeEntry("pivot_root", kPath, kPath, kGen, kGen, kGen, kGen), // 41 + MakeEntry("nfsservctl", UnknownArguments()), // 42 + MakeEntry("statfs", kPath, kGen, kGen, kGen, kGen, kGen), // 43 + MakeEntry("fstatfs", UnknownArguments()), // 44 + MakeEntry("truncate", kPath, kInt, kGen, kGen, kGen, kGen), // 45 + MakeEntry("ftruncate", UnknownArguments()), // 46 + MakeEntry("fallocate", UnknownArguments()), // 47 + MakeEntry("faccessat", kGen, kPath, kGen, kGen, kGen, kGen), // 48 + MakeEntry("chdir", kPath, kGen, kGen, kGen, kGen, kGen), // 49 + MakeEntry("fchdir", UnknownArguments()), // 50 + MakeEntry("chroot", kPath, kGen, kGen, kGen, kGen, kGen), // 51 + MakeEntry("fchmod", UnknownArguments()), // 52 + MakeEntry("fchmodat", kGen, kPath, kGen, kGen, kGen, kGen), // 53 + MakeEntry("fchownat", kGen, kPath, kGen, kGen, kGen, kGen), // 54 + MakeEntry("fchown", UnknownArguments()), // 55 + MakeEntry("openat", kGen, kPath, kOct, kHex, kGen, kGen), // 56 + MakeEntry("close", kInt, kGen, kGen, kGen, kGen, kGen), // 57 + MakeEntry("vhangup", UnknownArguments()), // 58 + MakeEntry("pipe2", UnknownArguments()), // 59 + MakeEntry("quotactl", kInt, kPath, kInt, kGen, kGen, kGen), // 60 + MakeEntry("getdents64", UnknownArguments()), // 61 + MakeEntry("lseek", UnknownArguments()), // 62 + MakeEntry("read", kInt, kHex, kInt, kGen, kGen, kGen), // 63 + MakeEntry("write", kInt, kHex, kInt, kGen, kGen, kGen), // 64 + MakeEntry("readv", UnknownArguments()), // 65 + MakeEntry("writev", UnknownArguments()), // 66 + MakeEntry("pread64", UnknownArguments()), // 67 + MakeEntry("pwrite64", UnknownArguments()), // 68 + MakeEntry("preadv", UnknownArguments()), // 69 + MakeEntry("pwritev", UnknownArguments()), // 70 + MakeEntry("sendfile", UnknownArguments()), // 71 + MakeEntry("pselect6", UnknownArguments()), // 72 + MakeEntry("ppoll", UnknownArguments()), // 73 + MakeEntry("signalfd4", UnknownArguments()), // 74 + MakeEntry("vmsplice", UnknownArguments()), // 75 + MakeEntry("splice", UnknownArguments()), // 76 + MakeEntry("tee", UnknownArguments()), // 77 + MakeEntry("readlinkat", kGen, kPath, kGen, kGen, kGen, kGen), // 78 + MakeEntry("newfstatat", kGen, kPath, kGen, kGen, kGen, kGen), // 79 + MakeEntry("fstat", kInt, kHex, kGen, kGen, kGen, kGen), // 80 + MakeEntry("sync", UnknownArguments()), // 81 + MakeEntry("fsync", UnknownArguments()), // 82 + MakeEntry("fdatasync", UnknownArguments()), // 83 + MakeEntry("sync_file_range", UnknownArguments()), // 84 + MakeEntry("timerfd_create", UnknownArguments()), // 85 + MakeEntry("timerfd_settime", UnknownArguments()), // 86 + MakeEntry("timerfd_gettime", UnknownArguments()), // 87 + MakeEntry("utimensat", UnknownArguments()), // 88 + MakeEntry("acct", kPath, kGen, kGen, kGen, kGen, kGen), // 89 + MakeEntry("capget", UnknownArguments()), // 90 + MakeEntry("capset", UnknownArguments()), // 91 + MakeEntry("personality", UnknownArguments()), // 92 + MakeEntry("exit", kInt, kGen, kGen, kGen, kGen, kGen), // 93 + MakeEntry("exit_group", kInt, kGen, kGen, kGen, kGen, kGen), // 94 + MakeEntry("waitid", UnknownArguments()), // 95 + MakeEntry("set_tid_address", kHex, kGen, kGen, kGen, kGen, kGen), // 96 + MakeEntry("unshare", UnknownArguments()), // 97 + MakeEntry("futex", UnknownArguments()), // 98 + MakeEntry("set_robust_list", UnknownArguments()), // 99 + MakeEntry("get_robust_list", UnknownArguments()), // 100 + MakeEntry("nanosleep", kHex, kHex, kGen, kGen, kGen, kGen), // 101 + MakeEntry("getitimer", UnknownArguments()), // 102 + MakeEntry("setitimer", UnknownArguments()), // 103 + MakeEntry("kexec_load", UnknownArguments()), // 104 + MakeEntry("init_module", UnknownArguments()), // 105 + MakeEntry("delete_module", UnknownArguments()), // 106 + MakeEntry("timer_create", UnknownArguments()), // 107 + MakeEntry("timer_gettime", UnknownArguments()), // 108 + MakeEntry("timer_getoverrun", UnknownArguments()), // 109 + MakeEntry("timer_settime", UnknownArguments()), // 110 + MakeEntry("timer_delete", UnknownArguments()), // 111 + MakeEntry("clock_settime", UnknownArguments()), // 112 + MakeEntry("clock_gettime", UnknownArguments()), // 113 + MakeEntry("clock_getres", UnknownArguments()), // 114 + MakeEntry("clock_nanosleep", UnknownArguments()), // 115 + MakeEntry("syslog", UnknownArguments()), // 116 + MakeEntry("ptrace", UnknownArguments()), // 117 + MakeEntry("sched_setparam", UnknownArguments()), // 118 + MakeEntry("sched_setscheduler", UnknownArguments()), // 119 + MakeEntry("sched_getscheduler", UnknownArguments()), // 120 + MakeEntry("sched_getparam", UnknownArguments()), // 121 + MakeEntry("sched_setaffinity", UnknownArguments()), // 122 + MakeEntry("sched_getaffinity", UnknownArguments()), // 123 + MakeEntry("sched_yield", UnknownArguments()), // 124 + MakeEntry("sched_get_priority_max", UnknownArguments()), // 125 + MakeEntry("sched_get_priority_min", UnknownArguments()), // 126 + MakeEntry("sched_rr_get_interval", UnknownArguments()), // 127 + MakeEntry("restart_syscall", UnknownArguments()), // 128 + MakeEntry("kill", kInt, kSignal, kGen, kGen, kGen, kGen), // 129 + MakeEntry("tkill", kInt, kSignal, kGen, kGen, kGen, kGen), // 130 + MakeEntry("tgkill", kInt, kInt, kSignal, kGen, kGen, kGen), // 131 + MakeEntry("sigaltstack", UnknownArguments()), // 132 + MakeEntry("rt_sigsuspend", UnknownArguments()), // 133 + MakeEntry("rt_sigaction", kSignal, kHex, kHex, kInt, kGen, kGen), // 134 + MakeEntry("rt_sigprocmask", UnknownArguments()), // 135 + MakeEntry("rt_sigpending", UnknownArguments()), // 136 + MakeEntry("rt_sigtimedwait", UnknownArguments()), // 137 + MakeEntry("rt_sigqueueinfo", UnknownArguments()), // 138 + MakeEntry("rt_sigreturn", UnknownArguments()), // 139 + MakeEntry("setpriority", UnknownArguments()), // 140 + MakeEntry("getpriority", UnknownArguments()), // 141 + MakeEntry("reboot", UnknownArguments()), // 142 + MakeEntry("setregid", UnknownArguments()), // 143 + MakeEntry("setgid", UnknownArguments()), // 144 + MakeEntry("setreuid", UnknownArguments()), // 145 + MakeEntry("setuid", UnknownArguments()), // 146 + MakeEntry("setresuid", UnknownArguments()), // 147 + MakeEntry("getresuid", UnknownArguments()), // 148 + MakeEntry("setresgid", UnknownArguments()), // 149 + MakeEntry("getresgid", UnknownArguments()), // 150 + MakeEntry("setfsuid", UnknownArguments()), // 151 + MakeEntry("setfsgid", UnknownArguments()), // 152 + MakeEntry("times", UnknownArguments()), // 153 + MakeEntry("setpgid", UnknownArguments()), // 154 + MakeEntry("getpgid", UnknownArguments()), // 155 + MakeEntry("getsid", UnknownArguments()), // 156 + MakeEntry("setsid", UnknownArguments()), // 157 + MakeEntry("getgroups", UnknownArguments()), // 158 + MakeEntry("setgroups", UnknownArguments()), // 159 + MakeEntry("uname", UnknownArguments()), // 160 + MakeEntry("sethostname", UnknownArguments()), // 161 + MakeEntry("setdomainname", UnknownArguments()), // 162 + MakeEntry("getrlimit", UnknownArguments()), // 163 + MakeEntry("setrlimit", UnknownArguments()), // 164 + MakeEntry("getrusage", UnknownArguments()), // 165 + MakeEntry("umask", kHex, kGen, kGen, kGen, kGen, kGen), // 166 + MakeEntry("prctl", kInt, kHex, kHex, kHex, kHex, kGen), // 167 + MakeEntry("getcpu", kHex, kHex, kHex, kGen, kGen, kGen), // 168 + MakeEntry("gettimeofday", kHex, kHex, kGen, kGen, kGen, kGen), // 169 + MakeEntry("settimeofday", kHex, kHex, kGen, kGen, kGen, kGen), // 170 + MakeEntry("adjtimex", UnknownArguments()), // 171 + MakeEntry("getpid", UnknownArguments()), // 172 + MakeEntry("getppid", UnknownArguments()), // 173 + MakeEntry("getuid", UnknownArguments()), // 174 + MakeEntry("geteuid", UnknownArguments()), // 175 + MakeEntry("getgid", UnknownArguments()), // 176 + MakeEntry("getegid", UnknownArguments()), // 177 + MakeEntry("gettid", UnknownArguments()), // 178 + MakeEntry("sysinfo", UnknownArguments()), // 179 + MakeEntry("mq_open", UnknownArguments()), // 180 + MakeEntry("mq_unlink", UnknownArguments()), // 181 + MakeEntry("mq_timedsend", UnknownArguments()), // 182 + MakeEntry("mq_timedreceive", UnknownArguments()), // 183 + MakeEntry("mq_notify", UnknownArguments()), // 184 + MakeEntry("mq_getsetattr", UnknownArguments()), // 185 + MakeEntry("msgget", UnknownArguments()), // 186 + MakeEntry("msgctl", UnknownArguments()), // 187 + MakeEntry("msgrcv", UnknownArguments()), // 188 + MakeEntry("msgsnd", UnknownArguments()), // 189 + MakeEntry("semget", UnknownArguments()), // 190 + MakeEntry("semctl", UnknownArguments()), // 191 + MakeEntry("semtimedop", UnknownArguments()), // 192 + MakeEntry("semop", UnknownArguments()), // 193 + MakeEntry("shmget", UnknownArguments()), // 194 + MakeEntry("shmctl", UnknownArguments()), // 195 + MakeEntry("shmat", UnknownArguments()), // 196 + MakeEntry("shmdt", UnknownArguments()), // 197 + MakeEntry("socket", kAddressFamily, kInt, kInt, kGen, kGen, kGen), // 198 + MakeEntry("socketpair", UnknownArguments()), // 199 + MakeEntry("bind", UnknownArguments()), // 200 + MakeEntry("listen", UnknownArguments()), // 201 + MakeEntry("accept", UnknownArguments()), // 202 + MakeEntry("connect", kInt, kSockaddr, kInt, kGen, kGen, kGen), // 203 + MakeEntry("getsockname", UnknownArguments()), // 204 + MakeEntry("getpeername", UnknownArguments()), // 205 + MakeEntry("sendto", kInt, kGen, kInt, kHex, kSockaddr, kInt), // 206 + MakeEntry("recvfrom", UnknownArguments()), // 207 + MakeEntry("setsockopt", UnknownArguments()), // 208 + MakeEntry("getsockopt", UnknownArguments()), // 209 + MakeEntry("shutdown", UnknownArguments()), // 210 + MakeEntry("sendmsg", kInt, kSockmsghdr, kHex, kGen, kGen, kGen), // 211 + MakeEntry("recvmsg", UnknownArguments()), // 212 + MakeEntry("readahead", UnknownArguments()), // 213 + MakeEntry("brk", kHex, kGen, kGen, kGen, kGen, kGen), // 214 + MakeEntry("munmap", kHex, kHex, kGen, kGen, kGen, kGen), // 215 + MakeEntry("mremap", UnknownArguments()), // 216 + MakeEntry("add_key", UnknownArguments()), // 217 + MakeEntry("request_key", UnknownArguments()), // 218 + MakeEntry("keyctl", UnknownArguments()), // 219 + MakeEntry("clone", kCloneFlag, kHex, kHex, kHex, kHex, kGen), // 220 + MakeEntry("execve", kPath, kHex, kHex, kGen, kGen, kGen), // 221 + MakeEntry("mmap", kHex, kInt, kHex, kHex, kInt, kInt), // 222 + MakeEntry("fadvise64", UnknownArguments()), // 223 + MakeEntry("swapon", kPath, kHex, kGen, kGen, kGen, kGen), // 224 + MakeEntry("swapoff", kPath, kGen, kGen, kGen, kGen, kGen), // 225 + MakeEntry("mprotect", kHex, kHex, kHex, kGen, kGen, kGen), // 226 + MakeEntry("msync", UnknownArguments()), // 227 + MakeEntry("mlock", UnknownArguments()), // 228 + MakeEntry("munlock", UnknownArguments()), // 229 + MakeEntry("mlockall", UnknownArguments()), // 230 + MakeEntry("munlockall", UnknownArguments()), // 231 + MakeEntry("mincore", UnknownArguments()), // 232 + MakeEntry("madvise", UnknownArguments()), // 233 + MakeEntry("remap_file_pages", UnknownArguments()), // 234 + MakeEntry("mbind", UnknownArguments()), // 235 + MakeEntry("get_mempolicy", UnknownArguments()), // 236 + MakeEntry("set_mempolicy", UnknownArguments()), // 237 + MakeEntry("migrate_pages", UnknownArguments()), // 238 + MakeEntry("move_pages", UnknownArguments()), // 239 + MakeEntry("rt_tgsigqueueinfo", UnknownArguments()), // 240 + MakeEntry("perf_event_open", UnknownArguments()), // 241 + MakeEntry("accept4", UnknownArguments()), // 242 + MakeEntry("recvmmsg", kInt, kHex, kHex, kHex, kGen, kGen), // 243 + SYSCALLS_UNUSED("UNUSED244"), // 244 + SYSCALLS_UNUSED("UNUSED245"), // 245 + SYSCALLS_UNUSED("UNUSED246"), // 246 + SYSCALLS_UNUSED("UNUSED247"), // 247 + SYSCALLS_UNUSED("UNUSED248"), // 248 + SYSCALLS_UNUSED("UNUSED249"), // 249 + SYSCALLS_UNUSED("UNUSED250"), // 250 + SYSCALLS_UNUSED("UNUSED251"), // 251 + SYSCALLS_UNUSED("UNUSED252"), // 252 + SYSCALLS_UNUSED("UNUSED253"), // 253 + SYSCALLS_UNUSED("UNUSED254"), // 254 + SYSCALLS_UNUSED("UNUSED255"), // 255 + SYSCALLS_UNUSED("UNUSED256"), // 256 + SYSCALLS_UNUSED("UNUSED257"), // 257 + SYSCALLS_UNUSED("UNUSED258"), // 258 + SYSCALLS_UNUSED("UNUSED259"), // 259 + MakeEntry("wait4", kInt, kHex, kHex, kHex, kGen, kGen), // 260 + MakeEntry("prlimit64", kInt, kInt, kHex, kHex, kGen, kGen), // 261 + MakeEntry("fanotify_init", kHex, kHex, kInt, kGen, kGen, kGen), // 262 + MakeEntry("fanotify_mark", kInt, kHex, kInt, kPath, kGen, kGen), // 263 + MakeEntry("name_to_handle_at", kInt, kGen, kHex, kHex, kHex, kGen), // 264 + MakeEntry("open_by_handle_at", kInt, kHex, kHex, kGen, kGen, kGen), // 265 + MakeEntry("clock_adjtime", kInt, kHex, kGen, kGen, kGen, kGen), // 266 + MakeEntry("syncfs", kInt, kGen, kGen, kGen, kGen, kGen), // 267 + MakeEntry("setns", kInt, kHex, kGen, kGen, kGen, kGen), // 268 + MakeEntry("sendmmsg", kInt, kHex, kInt, kHex, kGen, kGen), // 269 + MakeEntry("process_vm_readv", kInt, kHex, kInt, kHex, kInt, kInt), // 270 + MakeEntry("process_vm_writev", kInt, kHex, kInt, kHex, kInt, kInt), // 271 + MakeEntry("kcmp", kInt, kInt, kInt, kHex, kHex, kGen), // 272 + MakeEntry("finit_module", kInt, kPath, kHex, kGen, kGen, kGen), // 273 + MakeEntry("sched_setattr", UnknownArguments()), // 274 + MakeEntry("sched_getattr", UnknownArguments()), // 275 + MakeEntry("renameat2", kGen, kPath, kGen, kPath, kGen, kGen), // 276 + MakeEntry("seccomp", UnknownArguments()), // 277 + MakeEntry("getrandom", UnknownArguments()), // 278 + MakeEntry("memfd_create", UnknownArguments()), // 279 +}; #undef SYSCALLS_UNUSED00_99 #undef SYSCALLS_UNUSED50_99 @@ -1226,17 +1507,16 @@ constexpr SyscallTable::Entry kSyscallDataPPC64[] = { #undef SYSCALLS_UNUSED0_9 #undef SYSCALLS_UNUSED -SyscallTable SyscallTable::get(Syscall::CpuArch arch) { - switch (arch) { -#if defined(__x86_64__) - case Syscall::kX86_64: +SyscallTable SyscallTable::get(cpu::Architecture arch) { + switch (host_cpu::Architecture()) { + case cpu::kX8664: return SyscallTable(kSyscallDataX8664); - case Syscall::kX86_32: + case cpu::kX86: return SyscallTable(kSyscallDataX8632); -#elif defined(__powerpc64__) - case Syscall::kPPC_64: - return SyscallTable(kSyscallDataPPC64); -#endif + case cpu::kPPC64LE: + return SyscallTable(kSyscallDataPPC64LE); + case cpu::kArm64: + return SyscallTable(kSyscallDataArm64); default: return SyscallTable(); } diff --git a/sandboxed_api/sandbox2/syscall_defs.h b/sandboxed_api/sandbox2/syscall_defs.h index bef3d19..168ed1d 100644 --- a/sandboxed_api/sandbox2/syscall_defs.h +++ b/sandboxed_api/sandbox2/syscall_defs.h @@ -9,6 +9,7 @@ #include "absl/strings/string_view.h" #include "absl/types/span.h" +#include "sandboxed_api/sandbox2/config.h" #include "sandboxed_api/sandbox2/syscall.h" namespace sandbox2 { @@ -23,7 +24,7 @@ class SyscallTable { struct Entry; // Returns the syscall table for the architecture. - static SyscallTable get(Syscall::CpuArch arch); + static SyscallTable get(cpu::Architecture arch); int size() { return data_.size(); } diff --git a/sandboxed_api/sandbox2/syscall_test.cc b/sandboxed_api/sandbox2/syscall_test.cc index 1cb7228..2d13257 100644 --- a/sandboxed_api/sandbox2/syscall_test.cc +++ b/sandboxed_api/sandbox2/syscall_test.cc @@ -19,6 +19,7 @@ #include "gmock/gmock.h" #include "gtest/gtest.h" #include "absl/strings/str_cat.h" +#include "sandboxed_api/sandbox2/config.h" using ::testing::Eq; using ::testing::StartsWith; @@ -45,7 +46,7 @@ TEST(SyscallTest, Basic) { EXPECT_THAT(arg_desc[2], Eq("0x5 [5]")); EXPECT_THAT( syscall.GetDescription(), - Eq(absl::StrCat(Syscall::GetArchDescription(Syscall::GetHostArch()), + Eq(absl::StrCat(Syscall::GetArchDescription(host_cpu::Architecture()), " read [", __NR_read, "](0x1 [1], 0xbadbeef, 0x5 [5]) IP: 0, STACK: 0"))); } @@ -53,7 +54,7 @@ TEST(SyscallTest, Basic) { TEST(SyscallTest, Empty) { Syscall syscall; - EXPECT_THAT(syscall.arch(), Eq(Syscall::kUnknown)); + EXPECT_THAT(syscall.arch(), Eq(cpu::kUnknown)); EXPECT_THAT(syscall.GetName(), StartsWith("UNKNOWN")); EXPECT_THAT(syscall.GetArgumentsDescription().size(), Eq(Syscall::kMaxArgs)); } diff --git a/sandboxed_api/sandbox2/testcases/BUILD.bazel b/sandboxed_api/sandbox2/testcases/BUILD.bazel index 80a1441..307db47 100644 --- a/sandboxed_api/sandbox2/testcases/BUILD.bazel +++ b/sandboxed_api/sandbox2/testcases/BUILD.bazel @@ -41,7 +41,12 @@ STATIC_LINKOPTS = [ ] # TODO(https://github.com/bazelbuild/bazel/issues/8672): Remove this workaround -EXTRA_FULLY_STATIC_LINKOPTS = ["-l:libstdc++.a"] +# Change is scheduled for Bazel 4.0. Specifying +# `--incompatible_linkopts_to_linklibs` also works +EXTRA_FULLY_STATIC_LINKOPTS = [ + "-l:libstdc++.a", + "-l:libm.a", +] cc_binary( name = "abort", @@ -173,6 +178,7 @@ cc_binary( "fully_static_link", # link libc statically ], linkstatic = 1, # prefer static libraries + deps = ["//sandboxed_api/sandbox2:config"], ) # security: disable=cc-static-no-pie diff --git a/sandboxed_api/sandbox2/testcases/CMakeLists.txt b/sandboxed_api/sandbox2/testcases/CMakeLists.txt index 4af43ba..c57ffc5 100644 --- a/sandboxed_api/sandbox2/testcases/CMakeLists.txt +++ b/sandboxed_api/sandbox2/testcases/CMakeLists.txt @@ -159,6 +159,7 @@ set_target_properties(policy PROPERTIES ) target_link_libraries(policy PRIVATE sapi::base + sandbox2::config ${_sandbox2_fully_static_linkopts} ) diff --git a/sandboxed_api/sandbox2/testcases/policy.cc b/sandboxed_api/sandbox2/testcases/policy.cc index e1f851e..ae598cf 100644 --- a/sandboxed_api/sandbox2/testcases/policy.cc +++ b/sandboxed_api/sandbox2/testcases/policy.cc @@ -23,7 +23,9 @@ #include #include -#if defined(__x86_64__) +#include "sandboxed_api/sandbox2/config.h" + +#ifdef SAPI_X86_64 void TestAMD64SyscallMismatch() { int64_t result; @@ -53,7 +55,7 @@ void TestAMD64SyscallMismatchFs() { : "rax", "rbx", "rcx"); exit(-result); } -#endif // defined(__x86_64__) +#endif void TestPtrace() { ptrace(PTRACE_SEIZE, getppid(), 0, 0); @@ -97,14 +99,14 @@ int main(int argc, char** argv) { int testno = atoi(argv[1]); // NOLINT switch (testno) { -#if defined(__x86_64__) +#ifdef SAPI_X86_64 case 1: TestAMD64SyscallMismatch(); break; case 2: TestAMD64SyscallMismatchFs(); break; -#endif // defined(__x86_64__) +#endif case 3: TestPtrace(); break; diff --git a/sandboxed_api/sandbox2/util.cc b/sandboxed_api/sandbox2/util.cc index 24d4a36..c3d90ba 100644 --- a/sandboxed_api/sandbox2/util.cc +++ b/sandboxed_api/sandbox2/util.cc @@ -39,13 +39,13 @@ #include "absl/strings/str_format.h" #include "absl/strings/string_view.h" #include "absl/strings/strip.h" +#include "sandboxed_api/sandbox2/config.h" #include "sandboxed_api/sandbox2/util/fileops.h" #include "sandboxed_api/sandbox2/util/path.h" #include "sandboxed_api/sandbox2/util/strerror.h" #include "sandboxed_api/util/raw_logging.h" -namespace sandbox2 { -namespace util { +namespace sandbox2::util { void CharPtrArrToVecString(char* const* arr, std::vector* vec) { for (int i = 0; arr[i]; ++i) { @@ -126,13 +126,11 @@ ABSL_ATTRIBUTE_NO_SANITIZE_ADDRESS ABSL_ATTRIBUTE_NOINLINE pid_t CloneAndJump(int flags, jmp_buf* env_ptr) { uint8_t stack_buf[PTHREAD_STACK_MIN] ABSL_CACHELINE_ALIGNED; -#if defined(__x86_64__) || defined(__x86__) || defined(__i386__) || \ - defined(__powerpc64__) + static_assert( + host_cpu::IsX8664() || host_cpu::IsPPC64LE() || host_cpu::IsArm64(), + "Host CPU architecture not supported, see config.h"); // Stack grows down. void* stack = stack_buf + sizeof(stack_buf); -#else -#error "Architecture is not supported" -#endif int r; { r = clone(&ChildFunc, stack, flags, env_ptr, nullptr, nullptr, nullptr); @@ -182,9 +180,9 @@ bool CreateMemFd(int* fd, const char* name) { return true; } -sapi::StatusOr Communicate(const std::vector& argv, - const std::vector& envv, - std::string* output) { +absl::StatusOr Communicate(const std::vector& argv, + const std::vector& envv, + std::string* output) { int cout_pipe[2]; posix_spawn_file_actions_t action; @@ -280,7 +278,7 @@ std::string GetRlimitName(int resource) { } } -sapi::StatusOr ReadCPathFromPid(pid_t pid, uintptr_t ptr) { +absl::StatusOr ReadCPathFromPid(pid_t pid, uintptr_t ptr) { std::string path(PATH_MAX, '\0'); iovec local_iov[] = {{&path[0], path.size()}}; @@ -321,5 +319,4 @@ sapi::StatusOr ReadCPathFromPid(pid_t pid, uintptr_t ptr) { return path; } -} // namespace util -} // namespace sandbox2 +} // namespace sandbox2::util diff --git a/sandboxed_api/sandbox2/util.h b/sandboxed_api/sandbox2/util.h index 87fe31c..a925b02 100644 --- a/sandboxed_api/sandbox2/util.h +++ b/sandboxed_api/sandbox2/util.h @@ -25,7 +25,7 @@ #include #include "absl/base/macros.h" -#include "sandboxed_api/util/statusor.h" +#include "absl/status/statusor.h" namespace sandbox2 { namespace util { @@ -62,9 +62,9 @@ bool CreateMemFd(int* fd, const char* name = "buffer_file"); // Executes a the program given by argv and the specified environment and // captures any output to stdout/stderr. -sapi::StatusOr Communicate(const std::vector& argv, - const std::vector& envv, - std::string* output); +absl::StatusOr Communicate(const std::vector& argv, + const std::vector& envv, + std::string* output); // Returns signal description. std::string GetSignalName(int signo); @@ -74,7 +74,7 @@ std::string GetRlimitName(int resource); // Reads a path string (NUL-terminated, shorter than PATH_MAX) from another // process memory -sapi::StatusOr ReadCPathFromPid(pid_t pid, uintptr_t ptr); +absl::StatusOr ReadCPathFromPid(pid_t pid, uintptr_t ptr); } // namespace util } // namespace sandbox2 diff --git a/sandboxed_api/sandbox2/util/BUILD.bazel b/sandboxed_api/sandbox2/util/BUILD.bazel index d48d9a4..924e22e 100644 --- a/sandboxed_api/sandbox2/util/BUILD.bazel +++ b/sandboxed_api/sandbox2/util/BUILD.bazel @@ -139,9 +139,9 @@ cc_library( "//sandboxed_api/sandbox2:util", "//sandboxed_api/util:raw_logging", "//sandboxed_api/util:status", - "//sandboxed_api/util:statusor", "@com_google_absl//absl/base:endian", "@com_google_absl//absl/status", + "@com_google_absl//absl/status:statusor", "@com_google_absl//absl/strings", ], ) @@ -173,8 +173,8 @@ cc_library( deps = [ ":fileops", ":strerror", - "//sandboxed_api/util:statusor", "@com_google_absl//absl/status", + "@com_google_absl//absl/status:statusor", "@com_google_absl//absl/strings", ], ) @@ -199,8 +199,8 @@ cc_library( hdrs = ["maps_parser.h"], copts = sapi_platform_copts(), deps = [ - "//sandboxed_api/util:statusor", "@com_google_absl//absl/status", + "@com_google_absl//absl/status:statusor", "@com_google_absl//absl/strings", ], ) diff --git a/sandboxed_api/sandbox2/util/CMakeLists.txt b/sandboxed_api/sandbox2/util/CMakeLists.txt index 1e13da4..46af750 100644 --- a/sandboxed_api/sandbox2/util/CMakeLists.txt +++ b/sandboxed_api/sandbox2/util/CMakeLists.txt @@ -80,7 +80,6 @@ target_link_libraries(sandbox2_util_minielf PRIVATE sandbox2::util sapi::base sapi::raw_logging - sapi::statusor ) # sandboxed_api/sandbox2/util:temp_file @@ -95,7 +94,7 @@ target_link_libraries(sandbox2_util_temp_file sandbox2::strerror sapi::base PUBLIC absl::status - sapi::statusor + absl::statusor ) # sandboxed_api/sandbox2/util:maps_parser @@ -106,9 +105,9 @@ add_library(sandbox2_util_maps_parser STATIC add_library(sandbox2::maps_parser ALIAS sandbox2_util_maps_parser) target_link_libraries(sandbox2_util_maps_parser PRIVATE absl::status + absl::statusor absl::strings sapi::base - sapi::statusor ) # sandboxed_api/sandbox2/util:runfiles diff --git a/sandboxed_api/sandbox2/util/maps_parser.cc b/sandboxed_api/sandbox2/util/maps_parser.cc index d3040cc..6687989 100644 --- a/sandboxed_api/sandbox2/util/maps_parser.cc +++ b/sandboxed_api/sandbox2/util/maps_parser.cc @@ -15,11 +15,12 @@ #include "sandboxed_api/sandbox2/util/maps_parser.h" #include "absl/status/status.h" +#include "absl/status/statusor.h" #include "absl/strings/str_split.h" namespace sandbox2 { -sapi::StatusOr> ParseProcMaps( +absl::StatusOr> ParseProcMaps( const std::string& contents) { // Note: The format string // https://github.com/torvalds/linux/blob/v4.14/fs/proc/task_mmu.c#L289 diff --git a/sandboxed_api/sandbox2/util/maps_parser.h b/sandboxed_api/sandbox2/util/maps_parser.h index 862d3e7..ea0a419 100644 --- a/sandboxed_api/sandbox2/util/maps_parser.h +++ b/sandboxed_api/sandbox2/util/maps_parser.h @@ -19,7 +19,7 @@ #include #include -#include "sandboxed_api/util/statusor.h" +#include "absl/status/statusor.h" namespace sandbox2 { @@ -37,7 +37,7 @@ struct MapsEntry { std::string path; }; -sapi::StatusOr> ParseProcMaps( +absl::StatusOr> ParseProcMaps( const std::string& contents); } // namespace sandbox2 diff --git a/sandboxed_api/sandbox2/util/minielf.cc b/sandboxed_api/sandbox2/util/minielf.cc index 3eac979..737dc97 100644 --- a/sandboxed_api/sandbox2/util/minielf.cc +++ b/sandboxed_api/sandbox2/util/minielf.cc @@ -95,7 +95,7 @@ class ElfParser { static constexpr size_t kMaxInterpreterSize = 1000; ElfParser() = default; - sapi::StatusOr Parse(FILE* elf, uint32_t features); + absl::StatusOr Parse(FILE* elf, uint32_t features); private: // Endianess support functions @@ -133,16 +133,16 @@ class ElfParser { // Reads elf header. absl::Status ReadFileHeader(); // Reads a single elf program header. - sapi::StatusOr ReadProgramHeader(absl::string_view src); + absl::StatusOr ReadProgramHeader(absl::string_view src); // Reads all elf program headers. absl::Status ReadProgramHeaders(); // Reads a single elf section header. - sapi::StatusOr ReadSectionHeader(absl::string_view src); + absl::StatusOr ReadSectionHeader(absl::string_view src); // Reads all elf section headers. absl::Status ReadSectionHeaders(); // Reads contents of an elf section. - sapi::StatusOr ReadSectionContents(int idx); - sapi::StatusOr ReadSectionContents( + absl::StatusOr ReadSectionContents(int idx); + absl::StatusOr ReadSectionContents( const Elf64_Shdr& section_header); // Reads all symbols from symtab section. absl::Status ReadSymbolsFromSymtab(const Elf64_Shdr& symtab); @@ -219,8 +219,7 @@ absl::Status ElfParser::ReadFileHeader() { return absl::OkStatus(); } -sapi::StatusOr ElfParser::ReadSectionHeader( - absl::string_view src) { +absl::StatusOr ElfParser::ReadSectionHeader(absl::string_view src) { if (src.size() < sizeof(Elf64_Shdr)) { return absl::FailedPreconditionError( absl::StrCat("invalid section header data: got ", src.size(), @@ -267,7 +266,7 @@ absl::Status ElfParser::ReadSectionHeaders() { return absl::OkStatus(); } -sapi::StatusOr ElfParser::ReadSectionContents(int idx) { +absl::StatusOr ElfParser::ReadSectionContents(int idx) { if (idx < 0 || idx >= section_headers_.size()) { return absl::FailedPreconditionError( absl::StrCat("invalid section header index: ", idx)); @@ -275,7 +274,7 @@ sapi::StatusOr ElfParser::ReadSectionContents(int idx) { return ReadSectionContents(section_headers_.at(idx)); } -sapi::StatusOr ElfParser::ReadSectionContents( +absl::StatusOr ElfParser::ReadSectionContents( const Elf64_Shdr& section_header) { auto offset = section_header.sh_offset; if (offset > file_size_) { @@ -293,8 +292,7 @@ sapi::StatusOr ElfParser::ReadSectionContents( return rv; } -sapi::StatusOr ElfParser::ReadProgramHeader( - absl::string_view src) { +absl::StatusOr ElfParser::ReadProgramHeader(absl::string_view src) { if (src.size() < sizeof(Elf64_Phdr)) { return absl::FailedPreconditionError( absl::StrCat("invalid program header data: got ", src.size(), @@ -456,7 +454,7 @@ absl::Status ElfParser::ReadImportedLibrariesFromDynamic( return absl::OkStatus(); } -sapi::StatusOr ElfParser::Parse(FILE* elf, uint32_t features) { +absl::StatusOr ElfParser::Parse(FILE* elf, uint32_t features) { elf_ = elf; // Basic sanity check. if (features & ~(ElfFile::kAll)) { @@ -513,8 +511,8 @@ sapi::StatusOr ElfParser::Parse(FILE* elf, uint32_t features) { return std::move(result_); } -sapi::StatusOr ElfFile::ParseFromFile(const std::string& filename, - uint32_t features) { +absl::StatusOr ElfFile::ParseFromFile(const std::string& filename, + uint32_t features) { std::unique_ptr elf{fopen(filename.c_str(), "r"), [](FILE* f) { fclose(f); }}; if (!elf) { diff --git a/sandboxed_api/sandbox2/util/minielf.h b/sandboxed_api/sandbox2/util/minielf.h index 247d271..027e21c 100644 --- a/sandboxed_api/sandbox2/util/minielf.h +++ b/sandboxed_api/sandbox2/util/minielf.h @@ -20,7 +20,7 @@ #include #include -#include "sandboxed_api/util/statusor.h" +#include "absl/status/statusor.h" namespace sandbox2 { @@ -33,8 +33,8 @@ class ElfFile { std::string name; }; - static sapi::StatusOr ParseFromFile(const std::string& filename, - uint32_t features); + static absl::StatusOr ParseFromFile(const std::string& filename, + uint32_t features); int64_t file_size() const { return file_size_; } const std::string& interpreter() const { return interpreter_; } diff --git a/sandboxed_api/sandbox2/util/temp_file.cc b/sandboxed_api/sandbox2/util/temp_file.cc index 9caac66..09a5a1b 100644 --- a/sandboxed_api/sandbox2/util/temp_file.cc +++ b/sandboxed_api/sandbox2/util/temp_file.cc @@ -22,6 +22,7 @@ #include #include +#include "absl/status/statusor.h" #include "absl/strings/str_cat.h" #include "sandboxed_api/sandbox2/util/fileops.h" #include "sandboxed_api/sandbox2/util/strerror.h" @@ -32,7 +33,7 @@ namespace { constexpr absl::string_view kMktempSuffix = "XXXXXX"; } // namespace -sapi::StatusOr> CreateNamedTempFile( +absl::StatusOr> CreateNamedTempFile( absl::string_view prefix) { std::string name_template = absl::StrCat(prefix, kMktempSuffix); int fd = mkstemp(&name_template[0]); @@ -42,7 +43,7 @@ sapi::StatusOr> CreateNamedTempFile( return std::pair{std::move(name_template), fd}; } -sapi::StatusOr CreateNamedTempFileAndClose( +absl::StatusOr CreateNamedTempFileAndClose( absl::string_view prefix) { auto result_or = CreateNamedTempFile(prefix); if (result_or.ok()) { @@ -55,7 +56,7 @@ sapi::StatusOr CreateNamedTempFileAndClose( return result_or.status(); } -sapi::StatusOr CreateTempDir(absl::string_view prefix) { +absl::StatusOr CreateTempDir(absl::string_view prefix) { std::string name_template = absl::StrCat(prefix, kMktempSuffix); if (mkdtemp(&name_template[0]) == nullptr) { return absl::UnknownError(absl::StrCat("mkdtemp():", StrError(errno))); diff --git a/sandboxed_api/sandbox2/util/temp_file.h b/sandboxed_api/sandbox2/util/temp_file.h index ae8f986..8324a61 100644 --- a/sandboxed_api/sandbox2/util/temp_file.h +++ b/sandboxed_api/sandbox2/util/temp_file.h @@ -17,24 +17,24 @@ #include -#include "sandboxed_api/util/statusor.h" +#include "absl/status/statusor.h" namespace sandbox2 { // Creates a temporary file under a path starting with prefix. File is not // unlinked and its path is returned together with an open fd. -sapi::StatusOr> CreateNamedTempFile( +absl::StatusOr> CreateNamedTempFile( absl::string_view prefix); // Creates a temporary file under a path starting with prefix. File is not // unlinked and its path is returned. FD of the created file is closed just // after creation. -sapi::StatusOr CreateNamedTempFileAndClose( +absl::StatusOr CreateNamedTempFileAndClose( absl::string_view prefix); // Creates a temporary directory under a path starting with prefix. // Returns the path of the created directory. -sapi::StatusOr CreateTempDir(absl::string_view prefix); +absl::StatusOr CreateTempDir(absl::string_view prefix); } // namespace sandbox2 diff --git a/sandboxed_api/sandbox2/util_test.cc b/sandboxed_api/sandbox2/util_test.cc index 55de298..3fa21ff 100644 --- a/sandboxed_api/sandbox2/util_test.cc +++ b/sandboxed_api/sandbox2/util_test.cc @@ -23,8 +23,8 @@ #include "sandboxed_api/sandbox2/testing.h" #include "sandboxed_api/sandbox2/util/path.h" -using testing::Gt; -using testing::IsTrue; +using ::testing::Gt; +using ::testing::IsTrue; namespace sandbox2 { namespace util { diff --git a/sandboxed_api/sandbox2/violation.proto b/sandboxed_api/sandbox2/violation.proto index acd8a6f..235cf6a 100644 --- a/sandboxed_api/sandbox2/violation.proto +++ b/sandboxed_api/sandbox2/violation.proto @@ -25,7 +25,6 @@ enum PBViolationType { SYSCALL_ARCHITECTURE_MISMATCH = 3; } -// X86_64 not allowed (naming convention...) message RegisterX8664 { uint64 r15 = 1; uint64 r14 = 2; @@ -77,7 +76,6 @@ message RegisterPowerpc64 { uint64 zero3 = 17; } -// Deprecated. message RegisterAarch64 { repeated uint64 regs = 1; uint64 sp = 2; @@ -90,7 +88,7 @@ message RegisterValues { oneof register_values { RegisterX8664 register_x86_64 = 2; RegisterPowerpc64 register_powerpc64 = 3; - RegisterAarch64 register_aarch64 = 4; // Deprecated. + RegisterAarch64 register_aarch64 = 4; } } @@ -105,10 +103,10 @@ message SyscallDescription { } message FsDescription { - repeated string file_whitelist = 1; - repeated string symlink_whitelist = 2; + repeated string file_allowlist = 1; + repeated string symlink_allowlist = 2; repeated string file_greylist = 3; - repeated string file_blacklist = 4; + repeated string file_denylist = 4; } message PolicyBuilderDescription { @@ -125,7 +123,7 @@ message NamespaceDescription { message PolicyDescription { bytes user_bpf_policy = 1; reserved 2 to 5; - // This requires additional fields. (e.g. whitelisted syscall #s) + // This requires additional fields. (e.g. allowed syscall numbers) PolicyBuilderDescription policy_builder_description = 6; // namespace diff --git a/sandboxed_api/tools/clang_generator/emitter.cc b/sandboxed_api/tools/clang_generator/emitter.cc index 18fdcd9..03bb5e3 100644 --- a/sandboxed_api/tools/clang_generator/emitter.cc +++ b/sandboxed_api/tools/clang_generator/emitter.cc @@ -15,6 +15,7 @@ #include "sandboxed_api/tools/clang_generator/emitter.h" #include "absl/random/random.h" +#include "absl/status/statusor.h" #include "absl/strings/ascii.h" #include "absl/strings/escaping.h" #include "absl/strings/match.h" @@ -46,10 +47,10 @@ constexpr absl::string_view kHeaderProlog = #include "absl/base/macros.h" #include "absl/status/status.h" +#include "absl/status/statusor.h" #include "sandboxed_api/sandbox.h" #include "sandboxed_api/vars.h" #include "sandboxed_api/util/status_macros.h" -#include "sandboxed_api/util/statusor.h" )"; constexpr absl::string_view kHeaderEpilog = @@ -196,7 +197,7 @@ std::string PrintFunctionPrototype(const clang::FunctionDecl* decl) { return out; } -sapi::StatusOr EmitFunction(const clang::FunctionDecl* decl) { +absl::StatusOr EmitFunction(const clang::FunctionDecl* decl) { std::string out; absl::StrAppend(&out, "\n// ", PrintFunctionPrototype(decl), "\n"); const std::string function_name = decl->getNameAsString(); @@ -248,7 +249,7 @@ sapi::StatusOr EmitFunction(const clang::FunctionDecl* decl) { return out; } -sapi::StatusOr EmitHeader( +absl::StatusOr EmitHeader( std::vector functions, const QualTypeSet& types, const GeneratorOptions& options) { std::string out; diff --git a/sandboxed_api/tools/clang_generator/emitter.h b/sandboxed_api/tools/clang_generator/emitter.h index 42ba92d..78e1f12 100644 --- a/sandboxed_api/tools/clang_generator/emitter.h +++ b/sandboxed_api/tools/clang_generator/emitter.h @@ -18,12 +18,12 @@ #include #include "absl/status/status.h" +#include "absl/status/statusor.h" #include "absl/strings/string_view.h" #include "clang/AST/Decl.h" #include "clang/AST/Type.h" #include "sandboxed_api/tools/clang_generator/generator.h" #include "sandboxed_api/tools/clang_generator/types.h" -#include "sandboxed_api/util/statusor.h" namespace sapi { @@ -36,7 +36,7 @@ namespace sapi { std::string GetIncludeGuard(absl::string_view filename); // Outputs a formatted header for a list of functions and their related types. -sapi::StatusOr EmitHeader( +absl::StatusOr EmitHeader( std::vector functions, const QualTypeSet& types, const GeneratorOptions& options); diff --git a/sandboxed_api/tools/clang_generator/generator.cc b/sandboxed_api/tools/clang_generator/generator.cc index 508877e..90504eb 100644 --- a/sandboxed_api/tools/clang_generator/generator.cc +++ b/sandboxed_api/tools/clang_generator/generator.cc @@ -18,12 +18,12 @@ #include #include "absl/status/status.h" +#include "absl/status/statusor.h" #include "clang/Format/Format.h" #include "sandboxed_api/sandbox2/util/fileops.h" #include "sandboxed_api/tools/clang_generator/diagnostics.h" #include "sandboxed_api/tools/clang_generator/emitter.h" #include "sandboxed_api/util/status_macros.h" -#include "sandboxed_api/util/statusor.h" namespace sapi { namespace { @@ -67,8 +67,8 @@ bool GeneratorASTVisitor::VisitFunctionDecl(clang::FunctionDecl* decl) { namespace internal { -sapi::StatusOr ReformatGoogleStyle(const std::string& filename, - const std::string& code) { +absl::StatusOr ReformatGoogleStyle(const std::string& filename, + const std::string& code) { // Configure code style based on Google style, but enforce pointer alignment clang::format::FormatStyle style = clang::format::getGoogleStyle(clang::format::FormatStyle::LK_Cpp); diff --git a/sandboxed_api/tools/clang_generator/generator.h b/sandboxed_api/tools/clang_generator/generator.h index 24e2658..8732a26 100644 --- a/sandboxed_api/tools/clang_generator/generator.h +++ b/sandboxed_api/tools/clang_generator/generator.h @@ -20,13 +20,13 @@ #include "absl/container/flat_hash_set.h" #include "absl/memory/memory.h" #include "absl/status/status.h" +#include "absl/status/statusor.h" #include "clang/AST/ASTConsumer.h" #include "clang/AST/RecursiveASTVisitor.h" #include "clang/Frontend/CompilerInstance.h" #include "clang/Frontend/FrontendAction.h" #include "clang/Tooling/Tooling.h" #include "sandboxed_api/tools/clang_generator/types.h" -#include "sandboxed_api/util/statusor.h" namespace sapi { @@ -66,8 +66,8 @@ class GeneratorASTVisitor namespace internal { -sapi::StatusOr ReformatGoogleStyle(const std::string& filename, - const std::string& code); +absl::StatusOr ReformatGoogleStyle(const std::string& filename, + const std::string& code); } // namespace internal diff --git a/sandboxed_api/tools/clang_generator/types.cc b/sandboxed_api/tools/clang_generator/types.cc index 4d91499..8efb4ef 100644 --- a/sandboxed_api/tools/clang_generator/types.cc +++ b/sandboxed_api/tools/clang_generator/types.cc @@ -208,7 +208,7 @@ std::string MapQualTypeReturn(const clang::ASTContext& context, return "absl::Status"; } // Remove const qualifier like in MapQualType(). - return absl::StrCat("::sapi::StatusOr<", + return absl::StrCat("absl::StatusOr<", MaybeRemoveConst(context, qual).getAsString(), ">"); } diff --git a/sandboxed_api/tools/clang_generator/types.h b/sandboxed_api/tools/clang_generator/types.h index 82a4478..7947df3 100644 --- a/sandboxed_api/tools/clang_generator/types.h +++ b/sandboxed_api/tools/clang_generator/types.h @@ -68,7 +68,7 @@ std::string MapQualTypeParameter(const clang::ASTContext& context, // Maps a qualified type used as a function return type to a type name // compatible with the generated Sandboxed API. Uses MapQualTypeParameter() and -// wraps the type in a sapi::StatusOr<> if qual is non-void. Otherwise returns +// wraps the type in an absl::StatusOr<> if qual is non-void. Otherwise returns // absl::Status. std::string MapQualTypeReturn(const clang::ASTContext& context, clang::QualType qual); diff --git a/sandboxed_api/tools/generator2/code.py b/sandboxed_api/tools/generator2/code.py index fb31101..f483d5e 100644 --- a/sandboxed_api/tools/generator2/code.py +++ b/sandboxed_api/tools/generator2/code.py @@ -84,6 +84,7 @@ def _stringify_tokens(tokens, separator='\n'): TYPE_MAPPING = { cindex.TypeKind.VOID: '::sapi::v::Void', cindex.TypeKind.CHAR_S: '::sapi::v::Char', + cindex.TypeKind.CHAR_U: '::sapi::v::UChar', cindex.TypeKind.INT: '::sapi::v::Int', cindex.TypeKind.UINT: '::sapi::v::UInt', cindex.TypeKind.LONG: '::sapi::v::Long', @@ -461,7 +462,7 @@ class ReturnType(ArgumentType): """Class representing function return type. Attributes: - return_type: sapi::StatusOr where T is original return type, or + return_type: absl::StatusOr where T is original return type, or absl::Status for functions returning void """ @@ -474,7 +475,7 @@ class ReturnType(ArgumentType): """Returns function return type prepared from the type.""" # TODO(szwl): const ptrs do not play well with SAPI C++ API... spelling = self._clang_type.spelling.replace('const', '') - return_type = 'sapi::StatusOr<{}>'.format(spelling) + return_type = 'absl::StatusOr<{}>'.format(spelling) return_type = 'absl::Status' if self.is_void() else return_type return return_type diff --git a/sandboxed_api/tools/generator2/code_test_util.py b/sandboxed_api/tools/generator2/code_test_util.py index 1fd5f9d..82ee3a6 100644 --- a/sandboxed_api/tools/generator2/code_test_util.py +++ b/sandboxed_api/tools/generator2/code_test_util.py @@ -32,7 +32,7 @@ class TestApi { ::sapi::Sandbox* sandbox() const { return sandbox_; } // int function_a(int, int) - sapi::StatusOr function_a(int x, int y) { + absl::StatusOr function_a(int x, int y) { ::sapi::v::Int ret; ::sapi::v::Int x_((x)); ::sapi::v::Int y_((y)); @@ -42,7 +42,7 @@ class TestApi { } // int types_1(bool, unsigned char, char, unsigned short, short) - sapi::StatusOr types_1(bool a0, unsigned char a1, char a2, unsigned short a3, short a4) { + absl::StatusOr types_1(bool a0, unsigned char a1, char a2, unsigned short a3, short a4) { ::sapi::v::Int ret; ::sapi::v::Bool a0_((a0)); ::sapi::v::UChar a1_((a1)); @@ -55,7 +55,7 @@ class TestApi { } // int types_2(int, unsigned int, long, unsigned long) - sapi::StatusOr types_2(int a0, unsigned int a1, long a2, unsigned long a3) { + absl::StatusOr types_2(int a0, unsigned int a1, long a2, unsigned long a3) { ::sapi::v::Int ret; ::sapi::v::Int a0_((a0)); ::sapi::v::UInt a1_((a1)); @@ -67,7 +67,7 @@ class TestApi { } // int types_3(long long, unsigned long long, float, double) - sapi::StatusOr types_3(long long a0, unsigned long long a1, float a2, double a3) { + absl::StatusOr types_3(long long a0, unsigned long long a1, float a2, double a3) { ::sapi::v::Int ret; ::sapi::v::LLong a0_((a0)); ::sapi::v::ULLong a1_((a1)); @@ -79,7 +79,7 @@ class TestApi { } // int types_4(signed char, short, int, long) - sapi::StatusOr types_4(signed char a0, short a1, int a2, long a3) { + absl::StatusOr types_4(signed char a0, short a1, int a2, long a3) { ::sapi::v::Int ret; ::sapi::v::SChar a0_((a0)); ::sapi::v::Short a1_((a1)); @@ -91,7 +91,7 @@ class TestApi { } // int types_5(long long, long double) - sapi::StatusOr types_5(long long a0, long double a1) { + absl::StatusOr types_5(long long a0, long double a1) { ::sapi::v::Int ret; ::sapi::v::LLong a0_((a0)); ::sapi::v::Reg a1_((a1)); @@ -136,7 +136,7 @@ class TestApi { ::sapi::Sandbox* sandbox() const { return sandbox_; } // uint function(uintp) - sapi::StatusOr function(::sapi::v::Ptr* a) { + absl::StatusOr function(::sapi::v::Ptr* a) { ::sapi::v::UInt ret; SAPI_RETURN_IF_ERROR(sandbox_->Call("function", &ret, a)); @@ -173,7 +173,7 @@ class TestApi { ::sapi::Sandbox* sandbox() const { return sandbox_; } // ProcessStatus ProcessDatapoint(ProcessStatus) - sapi::StatusOr ProcessDatapoint(ProcessStatus status) { + absl::StatusOr ProcessDatapoint(ProcessStatus status) { ::sapi::v::IntBase ret; ::sapi::v::IntBase status_((status)); diff --git a/sandboxed_api/transaction.cc b/sandboxed_api/transaction.cc index 4ecf9a6..a17c148 100644 --- a/sandboxed_api/transaction.cc +++ b/sandboxed_api/transaction.cc @@ -26,12 +26,10 @@ absl::Status TransactionBase::RunTransactionFunctionInSandbox( // Set the wall-time limit for this transaction run, and clean it up // afterwards, no matter what the result. - SAPI_RETURN_IF_ERROR(sandbox_->SetWallTimeLimit(GetTimeLimit())); + SAPI_RETURN_IF_ERROR(sandbox_->SetWallTimeLimit(absl::Seconds(GetTimeLimit()))); struct TimeCleanup { ~TimeCleanup() { - if (capture->sandbox_->IsActive()) { - capture->sandbox_->SetWallTimeLimit(0).IgnoreError(); - } + capture->sandbox_->SetWallTimeLimit(absl::ZeroDuration()).IgnoreError(); } TransactionBase* capture; } sandbox_cleanup = {this}; @@ -66,8 +64,11 @@ absl::Status TransactionBase::RunTransactionLoop( } TransactionBase::~TransactionBase() { - if (initialized_) { - Finish().IgnoreError(); + if (!initialized_) { + return; + } + if (absl::Status status = Finish(); !status.ok()) { + LOG(ERROR) << "Transaction finalizer returned an error: " << status; } } diff --git a/sandboxed_api/transaction.h b/sandboxed_api/transaction.h index 3ff00c2..b194a80 100644 --- a/sandboxed_api/transaction.h +++ b/sandboxed_api/transaction.h @@ -86,9 +86,7 @@ class TransactionBase { protected: explicit TransactionBase(std::unique_ptr sandbox) - : retry_count_(kDefaultRetryCount), - time_limit_(absl::ToTimeT(absl::UnixEpoch() + kDefaultTimeLimit)), - initialized_(false), + : time_limit_(absl::ToTimeT(absl::UnixEpoch() + kDefaultTimeLimit)), sandbox_(std::move(sandbox)) {} // Runs the main (retrying) transaction loop. @@ -115,14 +113,14 @@ class TransactionBase { virtual absl::Status Finish() { return absl::OkStatus(); } // Number of tries this transaction will be re-executed until it succeeds. - int retry_count_; + int retry_count_ = kDefaultRetryCount; // Time (wall-time) limit for a single Run() call (in seconds). 0 means: no // wall-time limit. time_t time_limit_; // Has Init() finished with success? - bool initialized_; + bool initialized_ = false; // The main sapi::Sandbox object. std::unique_ptr sandbox_; diff --git a/sandboxed_api/util/BUILD.bazel b/sandboxed_api/util/BUILD.bazel index 6f21685..d0f453c 100644 --- a/sandboxed_api/util/BUILD.bazel +++ b/sandboxed_api/util/BUILD.bazel @@ -45,20 +45,15 @@ cc_library( cc_library( name = "statusor", hdrs = ["statusor.h"], - copts = sapi_platform_copts(), - visibility = ["//visibility:public"], + deprecation = "Migrate to `absl::StatusOr`", deps = [ - ":raw_logging", - ":status", "@com_google_absl//absl/base:core_headers", - "@com_google_absl//absl/base:log_severity", - "@com_google_absl//absl/status", - "@com_google_absl//absl/types:variant", + "@com_google_absl//absl/status:statusor", ], ) -# gMock matchers for absl::Status and sapi::StatusOr and a gUnit printer -# extension for sapi::StatusOr. +# gMock matchers for absl::Status and absl::StatusOr and a gUnit printer +# extension. Adapted from the version in Asylo. cc_library( name = "status_matchers", testonly = 1, @@ -67,8 +62,8 @@ cc_library( visibility = ["//visibility:public"], deps = [ ":status", - ":statusor", "@com_google_absl//absl/status", + "@com_google_absl//absl/status:statusor", "@com_google_absl//absl/types:optional", "@com_google_googletest//:gtest", ], @@ -87,19 +82,6 @@ cc_test( ], ) -# Tests for the StatusOr template class. -cc_test( - name = "statusor_test", - srcs = ["statusor_test.cc"], - copts = sapi_platform_copts(), - deps = [ - ":status", - ":status_matchers", - ":statusor", - "@com_google_googletest//:gtest_main", - ], -) - # Tests for the Status macros. cc_test( name = "status_macros_test", @@ -108,9 +90,9 @@ cc_test( deps = [ ":status", ":status_matchers", - ":statusor", "@com_google_absl//absl/memory", "@com_google_absl//absl/status", + "@com_google_absl//absl/status:statusor", "@com_google_absl//absl/strings", "@com_google_googletest//:gtest_main", ], diff --git a/sandboxed_api/util/CMakeLists.txt b/sandboxed_api/util/CMakeLists.txt index 4bd6d25..9564188 100644 --- a/sandboxed_api/util/CMakeLists.txt +++ b/sandboxed_api/util/CMakeLists.txt @@ -48,11 +48,10 @@ add_library(sapi_util_statusor STATIC ) add_library(sapi::statusor ALIAS sapi_util_statusor) target_link_libraries(sapi_util_statusor PRIVATE - absl::base absl::core_headers - absl::variant - sapi::raw_logging - sapi::status + absl::status + absl::statusor + sapi::base ) # sandboxed_api/util:flag @@ -90,7 +89,6 @@ if(SAPI_ENABLE_TESTS) sapi::base PUBLIC absl::status sapi::status - sapi::statusor ) # sandboxed_api/util:status_test @@ -98,20 +96,24 @@ if(SAPI_ENABLE_TESTS) status_test.cc ) target_link_libraries(status_test PRIVATE + sapi::status sapi::status_matchers sapi::test_main + absl::status absl::type_traits ) gtest_discover_tests(status_test) - # sandboxed_api/util:statusor_test - add_executable(statusor_test - statusor_test.cc + # sandboxed_api/util:status_macros_test + add_executable(status_macros_test + status_macros_test.cc ) - target_link_libraries(statusor_test PRIVATE + target_link_libraries(status_macros_test PRIVATE sapi::status_matchers sapi::test_main + absl::status + absl::statusor absl::type_traits ) - gtest_discover_tests(statusor_test) + gtest_discover_tests(status_macros_test) endif() diff --git a/sandboxed_api/util/status.h b/sandboxed_api/util/status.h index 6390bc0..d9962fd 100644 --- a/sandboxed_api/util/status.h +++ b/sandboxed_api/util/status.h @@ -12,10 +12,6 @@ // See the License for the specific language governing permissions and // limitations under the License. -// This file and it's implementation provide a custom fork of -// util/task/status.h. This will become obsolete and will be replaced once -// Abseil releases absl::Status. - #ifndef THIRD_PARTY_SAPI_UTIL_STATUS_H_ #define THIRD_PARTY_SAPI_UTIL_STATUS_H_ diff --git a/sandboxed_api/util/status_macros.h b/sandboxed_api/util/status_macros.h index 8c7c011..d22215b 100644 --- a/sandboxed_api/util/status_macros.h +++ b/sandboxed_api/util/status_macros.h @@ -42,6 +42,6 @@ if (ABSL_PREDICT_FALSE(!statusor.ok())) { \ return statusor.status(); \ } \ - lhs = std::move(statusor).ValueOrDie(); + lhs = std::move(statusor).value(); #endif // THIRD_PARTY_SAPI_UTIL_STATUS_MACROS_H_ diff --git a/sandboxed_api/util/status_macros_test.cc b/sandboxed_api/util/status_macros_test.cc index f74c83c..647149b 100644 --- a/sandboxed_api/util/status_macros_test.cc +++ b/sandboxed_api/util/status_macros_test.cc @@ -20,10 +20,10 @@ #include "gtest/gtest.h" #include "absl/memory/memory.h" #include "absl/status/status.h" +#include "absl/status/statusor.h" #include "absl/strings/str_cat.h" #include "sandboxed_api/util/status.h" #include "sandboxed_api/util/status_matchers.h" -#include "sandboxed_api/util/statusor.h" namespace sapi { namespace { @@ -52,17 +52,17 @@ TEST(ReturnIfError, ReturnsOnErrorFromLambda) { TEST(AssignOrReturn, AssignsMultipleVariablesInSequence) { auto func = []() -> absl::Status { int value1; - SAPI_ASSIGN_OR_RETURN(value1, StatusOr(1)); + SAPI_ASSIGN_OR_RETURN(value1, absl::StatusOr(1)); EXPECT_EQ(1, value1); int value2; - SAPI_ASSIGN_OR_RETURN(value2, StatusOr(2)); + SAPI_ASSIGN_OR_RETURN(value2, absl::StatusOr(2)); EXPECT_EQ(2, value2); int value3; - SAPI_ASSIGN_OR_RETURN(value3, StatusOr(3)); + SAPI_ASSIGN_OR_RETURN(value3, absl::StatusOr(3)); EXPECT_EQ(3, value3); int value4; SAPI_ASSIGN_OR_RETURN(value4, - StatusOr(absl::UnknownError("EXPECTED"))); + absl::StatusOr(absl::UnknownError("EXPECTED"))); return absl::UnknownError(absl::StrCat("ERROR: assigned value ", value4)); }; @@ -72,11 +72,12 @@ TEST(AssignOrReturn, AssignsMultipleVariablesInSequence) { TEST(AssignOrReturn, AssignsRepeatedlyToSingleVariable) { auto func = []() -> absl::Status { int value = 1; - SAPI_ASSIGN_OR_RETURN(value, StatusOr(2)); + SAPI_ASSIGN_OR_RETURN(value, absl::StatusOr(2)); EXPECT_EQ(2, value); - SAPI_ASSIGN_OR_RETURN(value, StatusOr(3)); + SAPI_ASSIGN_OR_RETURN(value, absl::StatusOr(3)); EXPECT_EQ(3, value); - SAPI_ASSIGN_OR_RETURN(value, StatusOr(absl::UnknownError("EXPECTED"))); + SAPI_ASSIGN_OR_RETURN(value, + absl::StatusOr(absl::UnknownError("EXPECTED"))); return absl::UnknownError("ERROR"); }; @@ -87,7 +88,7 @@ TEST(AssignOrReturn, MovesUniquePtr) { auto func = []() -> absl::Status { std::unique_ptr ptr; SAPI_ASSIGN_OR_RETURN( - ptr, StatusOr>(absl::make_unique(1))); + ptr, absl::StatusOr>(absl::make_unique(1))); EXPECT_EQ(*ptr, 1); return absl::UnknownError("EXPECTED"); }; @@ -98,8 +99,8 @@ TEST(AssignOrReturn, MovesUniquePtr) { TEST(AssignOrReturn, DoesNotAssignUniquePtrOnErrorStatus) { auto func = []() -> absl::Status { std::unique_ptr ptr; - SAPI_ASSIGN_OR_RETURN( - ptr, StatusOr>(absl::UnknownError("EXPECTED"))); + SAPI_ASSIGN_OR_RETURN(ptr, absl::StatusOr>( + absl::UnknownError("EXPECTED"))); EXPECT_EQ(ptr, nullptr); return absl::OkStatus(); }; @@ -111,10 +112,10 @@ TEST(AssignOrReturn, MovesUniquePtrRepeatedlyToSingleVariable) { auto func = []() -> absl::Status { std::unique_ptr ptr; SAPI_ASSIGN_OR_RETURN( - ptr, StatusOr>(absl::make_unique(1))); + ptr, absl::StatusOr>(absl::make_unique(1))); EXPECT_EQ(*ptr, 1); SAPI_ASSIGN_OR_RETURN( - ptr, StatusOr>(absl::make_unique(2))); + ptr, absl::StatusOr>(absl::make_unique(2))); EXPECT_EQ(*ptr, 2); return absl::UnknownError("EXPECTED"); }; diff --git a/sandboxed_api/util/status_matchers.h b/sandboxed_api/util/status_matchers.h index c7ed2e5..f181fc6 100644 --- a/sandboxed_api/util/status_matchers.h +++ b/sandboxed_api/util/status_matchers.h @@ -19,9 +19,9 @@ #include "gmock/gmock.h" #include "absl/status/status.h" +#include "absl/status/statusor.h" #include "absl/types/optional.h" #include "sandboxed_api/util/status_macros.h" -#include "sandboxed_api/util/statusor.h" #define SAPI_ASSERT_OK_AND_ASSIGN(lhs, rexpr) \ SAPI_ASSERT_OK_AND_ASSIGN_IMPL( \ diff --git a/sandboxed_api/util/statusor.h b/sandboxed_api/util/statusor.h index f1f5be3..51bd488 100644 --- a/sandboxed_api/util/statusor.h +++ b/sandboxed_api/util/statusor.h @@ -12,220 +12,17 @@ // See the License for the specific language governing permissions and // limitations under the License. -// This file and it's implementation provide a custom fork of -// util/task/statusor.h. This will become obsolete and will be replaced once -// Abseil releases absl::Status. - #ifndef THIRD_PARTY_SAPI_UTIL_STATUSOR_H_ #define THIRD_PARTY_SAPI_UTIL_STATUSOR_H_ -#include -#include - -#include "absl/base/internal/raw_logging.h" #include "absl/base/attributes.h" -#include "absl/base/log_severity.h" -#include "absl/status/status.h" -#include "absl/types/variant.h" -#include "sandboxed_api/util/raw_logging.h" +#include "absl/status/statusor.h" namespace sapi { template -class ABSL_MUST_USE_RESULT StatusOr { - template - friend class StatusOr; - - public: - using element_type = T; - - explicit StatusOr() : variant_(absl::UnknownError("")) {} - - StatusOr(const StatusOr&) = default; - StatusOr& operator=(const StatusOr&) = default; - - StatusOr(StatusOr&&) = default; - StatusOr& operator=(StatusOr&&) = default; - - // Not implemented: - // template StatusOr(const StatusOr& other) - // template StatusOr(StatusOr&& other) - - template - StatusOr& operator=(const StatusOr& other) { - if (other.ok()) { - variant_ = other.value(); - } else { - variant_ = other.status(); - } - return *this; - } - - template - StatusOr& operator=(StatusOr&& other) { - if (other.ok()) { - variant_ = std::move(other).value(); - } else { - variant_ = std::move(other).status(); - } - return *this; - } - - StatusOr(const T& value) : variant_(value) {} - - StatusOr(const absl::Status& status) : variant_(status) { EnsureNotOk(); } - - // Not implemented: - // template StatusOr& operator=(U&& value) - - StatusOr(T&& value) : variant_(std::move(value)) {} - - StatusOr(absl::Status&& value) : variant_(std::move(value)) {} - - StatusOr& operator=(absl::Status&& status) { - variant_ = std::move(status); - EnsureNotOk(); - } - - template - explicit StatusOr(absl::in_place_t, Args&&... args) - : StatusOr(T(std::forward(args)...)) {} - - template - explicit StatusOr(absl::in_place_t, std::initializer_list ilist, - Args&&... args) - : StatusOr(ilist, U(std::forward(args)...)) {} - - explicit operator bool() const { return ok(); } - - ABSL_MUST_USE_RESULT bool ok() const { - return absl::holds_alternative(variant_); - } - - const absl::Status& status() const& { - static const auto* ok_status = new absl::Status(); - return ok() ? *ok_status : absl::get(variant_); - } - - absl::Status status() && { - return ok() ? absl::OkStatus() - : std::move(absl::get(variant_)); - } - - const T& value() const& { - EnsureOk(); - return absl::get(variant_); - } - - T& value() & { - EnsureOk(); - return absl::get(variant_); - } - - const T&& value() const&& { - EnsureOk(); - return absl::get(std::move(variant_)); - } - - T&& value() && { - EnsureOk(); - return absl::get(std::move(variant_)); - } - - const T& ValueOrDie() const& { - EnsureOk(); - return absl::get(variant_); - } - - T& ValueOrDie() & { - EnsureOk(); - return absl::get(variant_); - } - - T&& ValueOrDie() && { - EnsureOk(); - return absl::get(std::move(variant_)); - } - - const T& operator*() const& { - EnsureOk(); - return absl::get(variant_); - } - - T& operator*() & { - EnsureOk(); - return absl::get(variant_); - } - - const T&& operator*() const&& { - EnsureOk(); - return absl::get(std::move(variant_)); - } - - T&& operator*() && { - EnsureOk(); - return absl::get(std::move(variant_)); - } - - const T* operator->() const { - EnsureOk(); - return &absl::get(variant_); - } - - T* operator->() { - EnsureOk(); - return &absl::get(variant_); - } - - template - T value_or(U&& default_value) const& { - if (ok()) { - return absl::get(variant_); - } - return std::forward(default_value); - } - - template - T value_or(U&& default_value) && { - if (ok()) { - return absl::get(std::move(variant_)); - } - return std::forward(default_value); - } - - void IgnoreError() const { /* no-op */ - } - - template - T& emplace(Args&&... args) { - return variant_.template emplace(std::forward(args)...); - } - - template - T& emplace(std::initializer_list ilist, Args&&... args) { - return variant_.template emplace(ilist, std::forward(args)...); - } - - private: - void EnsureOk() const { - if (!ok()) { - // GoogleTest needs this exact error message for death tests to work. - SAPI_RAW_LOG(FATAL, - "Attempting to fetch value instead of handling error %s", - status().message()); - } - } - - void EnsureNotOk() const { - if (ok()) { - SAPI_RAW_LOG( - FATAL, - "An OK status is not a valid constructor argument to StatusOr"); - } - } - - absl::variant variant_; -}; +using StatusOr ABSL_DEPRECATED("Use absl::StatusOr instead") = + absl::StatusOr; } // namespace sapi diff --git a/sandboxed_api/util/statusor_test.cc b/sandboxed_api/util/statusor_test.cc deleted file mode 100644 index ba80ce1..0000000 --- a/sandboxed_api/util/statusor_test.cc +++ /dev/null @@ -1,407 +0,0 @@ -// Copyright 2019 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// This file is a custom fork of the version in Asylo. This will become obsolete -// and will be replaced once Abseil releases absl::Status. - -#include "sandboxed_api/util/statusor.h" - -#include -#include -#include - -#include "gmock/gmock.h" -#include "gtest/gtest.h" -#include "sandboxed_api/util/status_matchers.h" - -using ::testing::Eq; -using ::testing::IsFalse; -using ::testing::Not; -using ::testing::Pointee; - -namespace sapi { -namespace { - -constexpr auto kErrorCode = absl::StatusCode::kInvalidArgument; -constexpr char kErrorMessage[] = "Invalid argument"; - -const int kIntElement = 47; -constexpr char kStringElement[] = "47 is 42, corrected for inflation"; - -// A data type without a default constructor. -struct Foo { - int bar; - std::string baz; - - explicit Foo(int value) : bar(value), baz(kStringElement) {} -}; - -// A data type with dynamically-allocated data. -struct HeapAllocatedObject { - int* value; - - HeapAllocatedObject() { - value = new int; - *value = kIntElement; - } - - HeapAllocatedObject(const HeapAllocatedObject& other) { *this = other; } - - HeapAllocatedObject& operator=(const HeapAllocatedObject& other) { - value = new int; - *value = *other.value; - return *this; - } - - HeapAllocatedObject(HeapAllocatedObject&& other) { *this = std::move(other); } - - HeapAllocatedObject& operator=(HeapAllocatedObject&& other) { - value = other.value; - other.value = nullptr; - return *this; - } - - ~HeapAllocatedObject() { delete value; } -}; - -// Constructs a Foo. -struct FooCtor { - using value_type = Foo; - - Foo operator()() { return Foo(kIntElement); } -}; - -// Constructs a HeapAllocatedObject. -struct HeapAllocatedObjectCtor { - using value_type = HeapAllocatedObject; - - HeapAllocatedObject operator()() { return HeapAllocatedObject(); } -}; - -// Constructs an integer. -struct IntCtor { - using value_type = int; - - int operator()() { return kIntElement; } -}; - -// Constructs a string. -struct StringCtor { - using value_type = std::string; - - std::string operator()() { return std::string(kStringElement); } -}; - -// Constructs a vector of strings. -struct StringVectorCtor { - using value_type = std::vector; - - std::vector operator()() { - return {kStringElement, kErrorMessage}; - } -}; - -bool operator==(const Foo& lhs, const Foo& rhs) { - return (lhs.bar == rhs.bar) && (lhs.baz == rhs.baz); -} - -bool operator==(const HeapAllocatedObject& lhs, - const HeapAllocatedObject& rhs) { - return *lhs.value == *rhs.value; -} - -// Returns an rvalue reference to the StatusOr object pointed to by -// |statusor|. -template -StatusOr&& MoveStatusOr(StatusOr* statusor) { - return std::move(*statusor); -} - -// A test fixture is required for typed tests. -template -class StatusOrTest : public ::testing::Test {}; - -using TestTypes = ::testing::Types; -TYPED_TEST_SUITE(StatusOrTest, TestTypes); - -// Verify that the default constructor for StatusOr constructs an object with a -// non-ok status. -TYPED_TEST(StatusOrTest, ConstructorDefault) { - StatusOr statusor; - EXPECT_THAT(statusor.ok(), IsFalse()); - EXPECT_THAT(statusor.status().code(), Eq(absl::StatusCode::kUnknown)); -} - -// Verify that StatusOr can be constructed from a Status object. -TYPED_TEST(StatusOrTest, ConstructorStatus) { - StatusOr statusor( - absl::Status(kErrorCode, kErrorMessage)); - - EXPECT_THAT(statusor.ok(), IsFalse()); - EXPECT_THAT(statusor.status().ok(), IsFalse()); - EXPECT_THAT(statusor.status(), Eq(absl::Status(kErrorCode, kErrorMessage))); -} - -// Verify that StatusOr can be constructed from an object of its element type. -TYPED_TEST(StatusOrTest, ConstructorElementConstReference) { - auto value = TypeParam()(); - StatusOr statusor{value}; - - ASSERT_THAT(statusor, IsOk()); - ASSERT_THAT(statusor.status(), IsOk()); - EXPECT_THAT(statusor.ValueOrDie(), Eq(value)); -} - -// Verify that StatusOr can be constructed from an rvalue reference of an object -// of its element type. -TYPED_TEST(StatusOrTest, ConstructorElementRValue) { - auto value = TypeParam()(); - auto value_copy(value); - StatusOr statusor(std::move(value)); - - ASSERT_THAT(statusor, IsOk()); - ASSERT_THAT(statusor.status(), IsOk()); - - // Compare to a copy of the original value, since the original was moved. - EXPECT_THAT(statusor.ValueOrDie(), Eq(value_copy)); -} - -// Verify that StatusOr can be copy-constructed from a StatusOr with a non-ok -// status. -TYPED_TEST(StatusOrTest, CopyConstructorNonOkStatus) { - StatusOr statusor1 = - absl::Status(kErrorCode, kErrorMessage); - StatusOr statusor2(statusor1); - - EXPECT_THAT(statusor1.ok(), Eq(statusor2.ok())); - EXPECT_THAT(statusor1.status(), Eq(statusor2.status())); -} - -// Verify that StatusOr can be copy-constructed from a StatusOr with an ok -// status. -TYPED_TEST(StatusOrTest, CopyConstructorOkStatus) { - StatusOr statusor1{TypeParam()()}; - StatusOr statusor2{statusor1}; - - EXPECT_THAT(statusor1.ok(), Eq(statusor2.ok())); - ASSERT_THAT(statusor2, IsOk()); - EXPECT_THAT(statusor1.ValueOrDie(), Eq(statusor2.ValueOrDie())); -} - -// Verify that copy-assignment of a StatusOr with a non-ok is working as -// expected. -TYPED_TEST(StatusOrTest, CopyAssignmentNonOkStatus) { - StatusOr statusor1{ - absl::Status(kErrorCode, kErrorMessage)}; - StatusOr statusor2{TypeParam()()}; - - // Invoke the copy-assignment operator. - statusor2 = statusor1; - EXPECT_THAT(statusor1.ok(), Eq(statusor2.ok())); - EXPECT_THAT(statusor1.status(), Eq(statusor2.status())); -} - -// Verify that copy-assignment of a StatusOr with an ok status is working as -// expected. -TYPED_TEST(StatusOrTest, CopyAssignmentOkStatus) { - StatusOr statusor1{TypeParam()()}; - StatusOr statusor2{ - absl::Status(kErrorCode, kErrorMessage)}; - - // Invoke the copy-assignment operator. - statusor2 = statusor1; - EXPECT_THAT(statusor1.ok(), Eq(statusor2.ok())); - ASSERT_THAT(statusor2, IsOk()); - EXPECT_THAT(statusor1.ValueOrDie(), Eq(statusor2.ValueOrDie())); -} - -// Verify that StatusOr can be move-constructed from a StatusOr with a non-ok -// status. -TYPED_TEST(StatusOrTest, MoveConstructorNonOkStatus) { - absl::Status status(kErrorCode, kErrorMessage); - StatusOr statusor1(status); - StatusOr statusor2(std::move(statusor1)); - - // Verify that the status of the donor object was updated. - EXPECT_THAT(statusor1.ok(), IsFalse()); // NOLINT - EXPECT_THAT(statusor1.status(), StatusIs(absl::StatusCode::kInternal)); - - // Verify that the destination object contains the status previously held by - // the donor. - EXPECT_THAT(statusor2.ok(), IsFalse()); - EXPECT_THAT(statusor2.status(), Eq(status)); -} - -// Verify that StatusOr can be move-constructed from a StatusOr with an ok -// status. -TYPED_TEST(StatusOrTest, MoveConstructorOkStatus) { - auto value = TypeParam()(); - StatusOr statusor1(value); - StatusOr statusor2(std::move(statusor1)); - - // The destination object should possess the value previously held by the - // donor. - ASSERT_THAT(statusor2, IsOk()); - EXPECT_THAT(statusor2.ValueOrDie(), Eq(value)); -} - -// Verify that move-assignment from a StatusOr with a non-ok status is working -// as expected. -TYPED_TEST(StatusOrTest, MoveAssignmentOperatorNonOkStatus) { - absl::Status status(kErrorCode, kErrorMessage); - StatusOr statusor1(status); - StatusOr statusor2{TypeParam()()}; - - // Invoke the move-assignment operator. - statusor2 = std::move(statusor1); - - // Verify that the status of the donor object was updated. - EXPECT_THAT(statusor1.ok(), IsFalse()); // NOLINT - EXPECT_THAT(statusor1.status(), StatusIs(absl::StatusCode::kInternal)); - - // Verify that the destination object contains the status previously held by - // the donor. - EXPECT_THAT(statusor2.ok(), IsFalse()); - EXPECT_THAT(statusor2.status(), Eq(status)); -} - -// Verify that move-assignment from a StatusOr with an ok status is working as -// expected. -TYPED_TEST(StatusOrTest, MoveAssignmentOperatorOkStatus) { - auto value = TypeParam()(); - StatusOr statusor1(value); - StatusOr statusor2( - absl::Status(kErrorCode, kErrorMessage)); - - // Invoke the move-assignment operator. - statusor2 = std::move(statusor1); - - // The destination object should possess the value previously held by the - // donor. - ASSERT_THAT(statusor2, IsOk()); - EXPECT_THAT(statusor2.ValueOrDie(), Eq(value)); -} - -// Verify that the sapi::IsOk() gMock matcher works with StatusOr. -TYPED_TEST(StatusOrTest, IsOkMatcher) { - auto value = TypeParam()(); - StatusOr statusor(value); - - EXPECT_THAT(statusor, IsOk()); - - statusor = StatusOr( - absl::Status(kErrorCode, kErrorMessage)); - EXPECT_THAT(statusor, Not(IsOk())); -} - -// Tests for move-only types. These tests use std::unique_ptr<> as the -// test type, since it is valuable to support this type in the Asylo infra. -// These tests are not part of the typed test suite for the following reasons: -// * std::unique_ptr<> cannot be used as a type in tests that expect -// the test type to support copy operations. -// * std::unique_ptr<> provides an equality operator that checks equality of -// the underlying ptr. Consequently, it is difficult to generalize existing -// tests that verify ValueOrDie() functionality using equality comparisons. - -// Verify that a StatusOr object can be constructed from a move-only type. -TEST(StatusOrTest, InitializationMoveOnlyType) { - auto* str = new std::string(kStringElement); - std::unique_ptr value(str); - StatusOr> statusor(std::move(value)); - - ASSERT_THAT(statusor, IsOk()); - EXPECT_THAT(statusor.ValueOrDie().get(), Eq(str)); -} - -// Verify that a StatusOr object can be move-constructed from a move-only type. -TEST(StatusOrTest, MoveConstructorMoveOnlyType) { - auto* str = new std::string(kStringElement); - std::unique_ptr value(str); - StatusOr> statusor1(std::move(value)); - StatusOr> statusor2(std::move(statusor1)); - - // The destination object should possess the value previously held by the - // donor. - ASSERT_THAT(statusor2, IsOk()); - EXPECT_THAT(statusor2.ValueOrDie().get(), Eq(str)); -} - -// Verify that a StatusOr object can be move-assigned to from a StatusOr object -// containing a move-only type. -TEST(StatusOrTest, MoveAssignmentMoveOnlyType) { - auto* str = new std::string(kStringElement); - std::unique_ptr value(str); - StatusOr> statusor1(std::move(value)); - StatusOr> statusor2( - absl::Status(kErrorCode, kErrorMessage)); - - // Invoke the move-assignment operator. - statusor2 = std::move(statusor1); - - // The destination object should possess the value previously held by the - // donor. - ASSERT_THAT(statusor2, IsOk()); - EXPECT_THAT(statusor2.ValueOrDie().get(), Eq(str)); -} - -// Verify that a value can be moved out of a StatusOr object via ValueOrDie(). -TEST(StatusOrTest, ValueOrDieMovedValue) { - auto* str = new std::string(kStringElement); - std::unique_ptr value(str); - StatusOr> statusor(std::move(value)); - - std::unique_ptr moved_value = std::move(statusor).ValueOrDie(); - EXPECT_THAT(moved_value.get(), Eq(str)); - EXPECT_THAT(*moved_value, Eq(kStringElement)); -} - -TEST(StatusOrTest, MapToStatusOrUniquePtr) { - // A reduced version of a problematic type found in the wild. All of the - // operations below should compile. - using MapType = std::map>>; - - MapType a; - - // Move-construction - MapType b(std::move(a)); - - // Move-assignment - a = std::move(b); -} - -TEST(StatusOrTest, ValueOrOk) { - const StatusOr status_or = 0; - EXPECT_EQ(status_or.value_or(-1), 0); -} - -TEST(StatusOrTest, ValueOrDefault) { - const StatusOr status_or = absl::CancelledError(); - EXPECT_EQ(status_or.value_or(-1), -1); -} - -TEST(StatusOrTest, MoveOnlyValueOrOk) { - EXPECT_THAT(StatusOr>(absl::make_unique(0)) - .value_or(absl::make_unique(-1)), - Pointee(0)); -} - -TEST(StatusOr, MoveOnlyValueOrDefault) { - EXPECT_THAT(StatusOr>(absl::CancelledError()) - .value_or(absl::make_unique(-1)), - Pointee(-1)); -} - -} // namespace -} // namespace sapi diff --git a/sandboxed_api/var_array.h b/sandboxed_api/var_array.h index 60968ba..fcf2da9 100644 --- a/sandboxed_api/var_array.h +++ b/sandboxed_api/var_array.h @@ -15,9 +15,11 @@ #ifndef SANDBOXED_API_VAR_ARRAY_H_ #define SANDBOXED_API_VAR_ARRAY_H_ +#include #include #include +#include #include "absl/base/macros.h" #include "absl/status/status.h" #include "absl/strings/str_cat.h" @@ -39,19 +41,19 @@ class Array : public Var, public Pointable { nelem_(nelem), total_size_(nelem_ * sizeof(T)), buffer_owned_(false) { - SetLocal(const_cast(reinterpret_cast(arr_))); + SetLocal(const_cast*>(arr_)); } // The array is allocated and owned by this object. explicit Array(size_t nelem) - : arr_(static_cast(malloc(sizeof(T) * nelem))), - nelem_(nelem), - total_size_(nelem_ * sizeof(T)), - buffer_owned_(true) { - SetLocal(const_cast(reinterpret_cast(arr_))); + : nelem_(nelem), total_size_(nelem_ * sizeof(T)), buffer_owned_(true) { + void* storage = malloc(sizeof(T) * nelem); + CHECK(storage != nullptr); + SetLocal(storage); + arr_ = static_cast(storage); } virtual ~Array() { if (buffer_owned_) { - free(const_cast(reinterpret_cast(arr_))); + free(const_cast*>(arr_)); } } @@ -121,7 +123,7 @@ class Array : public Var, public Pointable { arr_ = static_cast(new_addr); total_size_ = size; nelem_ = size / sizeof(T); - SetLocal(arr_); + SetLocal(new_addr); return absl::OkStatus(); } @@ -142,7 +144,7 @@ class Array : public Var, public Pointable { class CStr : public Array { public: explicit CStr(char* cstr) : Array(strlen(cstr) + 1) { - strcpy(this->GetData(), cstr); // NOLINT + std::copy(cstr, cstr + GetNElem(), GetData()); } std::string ToString() const final { @@ -159,10 +161,7 @@ class ConstCStr : public Array { : Array(cstr, strlen(cstr) + 1) {} std::string ToString() const final { - if (GetData() == nullptr) { - return "CStr: [nullptr]"; - } - return absl::StrCat("CStr: len(w/o NUL):", strlen(GetData()), ", ['", + return absl::StrCat("ConstCStr: len(w/o NUL):", strlen(GetData()), ", ['", GetData(), "']"); } }; diff --git a/sandboxed_api/var_proto.h b/sandboxed_api/var_proto.h index 4d1dac2..ffa8128 100644 --- a/sandboxed_api/var_proto.h +++ b/sandboxed_api/var_proto.h @@ -41,7 +41,7 @@ class Proto : public Pointable, public Var { explicit Proto(const T& proto) : wrapped_var_(SerializeProto(proto).value()) {} - static sapi::StatusOr> FromMessage(const T& proto) { + static absl::StatusOr> FromMessage(const T& proto) { SAPI_ASSIGN_OR_RETURN(std::vector len_val, SerializeProto(proto)); return Proto(len_val); } @@ -59,7 +59,7 @@ class Proto : public Pointable, public Var { void* GetLocal() const override { return wrapped_var_.GetLocal(); } // Returns a copy of the stored protobuf object. - sapi::StatusOr GetMessage() const { + absl::StatusOr GetMessage() const { return DeserializeProto( reinterpret_cast(wrapped_var_.GetData()), wrapped_var_.GetDataSize());